Refactor backup system and migrate from submodule to shadow dependencies
- Remove s3-lite submodule and replace with AWS SDK and shadow plugin - Implement comprehensive backup commands with list, delete, create, restore, and suspend functionality - Add bundle format with compression support using Zstd and gunzip optimization - Implement local and S3 backup strategies with configurable options - Add backup tracking system with JSON persistence - Improve world directory handling and file exclusion logic - Add standalone bundler tool for bundle creation/extraction - Update Gradle configuration with shadow jar support and dependency management - Enhance configuration system with additional options for compression and temporary directories
This commit is contained in:
1 parent
6a2642ab93
commit
fb42be8155
21 files changed
+746
-183
No files matched your search
+71
-53
@@ -1,94 +1,112 @@
|
||||
plugins {
|
||||
id 'fabric-loom' version "${loom_version}"
|
||||
id 'maven-publish'
|
||||
id("io.freefair.lombok") version "9.0.0-rc2"
|
||||
|
||||
id 'fabric-loom' version "${loom_version}"
|
||||
id 'maven-publish'
|
||||
id("io.freefair.lombok") version "9.0.0-rc2"
|
||||
id("com.gradleup.shadow") version '9.1.0'
|
||||
}
|
||||
|
||||
version = project.mod_version
|
||||
group = project.maven_group
|
||||
|
||||
base {
|
||||
archivesName = project.archives_base_name
|
||||
archivesName = project.archives_base_name
|
||||
}
|
||||
|
||||
repositories {
|
||||
// Add repositories to retrieve artifacts from in here.
|
||||
// You should only use this when depending on other mods because
|
||||
// Loom adds the essential maven repositories to download Minecraft and libraries from automatically.
|
||||
// See https://docs.gradle.org/current/userguide/declaring_repositories.html
|
||||
// for more information about repositories.
|
||||
// Add repositories to retrieve artifacts from in here.
|
||||
// You should only use this when depending on other mods because
|
||||
// Loom adds the essential maven repositories to download Minecraft and libraries from automatically.
|
||||
// See https://docs.gradle.org/current/userguide/declaring_repositories.html
|
||||
// for more information about repositories.
|
||||
maven {
|
||||
name = "Kalculos"
|
||||
url = "https://repo.sfclub.cc/snapshots"
|
||||
}
|
||||
}
|
||||
|
||||
loom {
|
||||
splitEnvironmentSourceSets()
|
||||
splitEnvironmentSourceSets()
|
||||
|
||||
mods {
|
||||
"modid" {
|
||||
sourceSet sourceSets.main
|
||||
sourceSet sourceSets.client
|
||||
}
|
||||
}
|
||||
mods {
|
||||
"justbackup" {
|
||||
sourceSet sourceSets.main
|
||||
sourceSet sourceSets.client
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
dependencies {
|
||||
// To change the versions see the gradle.properties file
|
||||
minecraft "com.mojang:minecraft:${project.minecraft_version}"
|
||||
mappings "net.fabricmc:yarn:${project.yarn_mappings}:v2"
|
||||
modImplementation "net.fabricmc:fabric-loader:${project.loader_version}"
|
||||
// To change the versions see the gradle.properties file
|
||||
minecraft "com.mojang:minecraft:${project.minecraft_version}"
|
||||
mappings "net.fabricmc:yarn:${project.yarn_mappings}:v2"
|
||||
modImplementation "net.fabricmc:fabric-loader:${project.loader_version}"
|
||||
|
||||
// Fabric API. This is technically optional, but you probably want it anyway.
|
||||
modImplementation "net.fabricmc.fabric-api:fabric-api:${project.fabric_version}"
|
||||
implementation("io.minio:minio:8.5.17")
|
||||
implementation 'com.github.luben:zstd-jni:1.5.7-4'
|
||||
// Fabric API. This is technically optional, but you probably want it anyway.
|
||||
modImplementation "net.fabricmc.fabric-api:fabric-api:${project.fabric_version}"
|
||||
// https://mvnrepository.com/artifact/software.amazon.awssdk/s3
|
||||
shadow 'software.amazon.awssdk:s3:2.34.0'
|
||||
shadow("io.ib67.kiwi:lang:1.2.1-SNAPSHOT")
|
||||
shadow 'com.github.luben:zstd-jni:1.5.7-4'
|
||||
}
|
||||
|
||||
configurations.implementation.extendsFrom(configurations.shadow)
|
||||
shadowJar {
|
||||
configurations = [project.configurations.shadow]
|
||||
exclude("META-INF")
|
||||
//relocate 'com.google', 'sfcraft.relocation.google'
|
||||
}
|
||||
|
||||
processResources {
|
||||
inputs.property "version", project.version
|
||||
inputs.property "version", project.version
|
||||
|
||||
filesMatching("fabric.mod.json") {
|
||||
expand "version": inputs.properties.version
|
||||
}
|
||||
filesMatching("fabric.mod.json") {
|
||||
expand "version": inputs.properties.version
|
||||
}
|
||||
}
|
||||
|
||||
tasks.withType(JavaCompile).configureEach {
|
||||
it.options.release = 21
|
||||
it.options.release = 21
|
||||
}
|
||||
|
||||
java {
|
||||
// Loom will automatically attach sourcesJar to a RemapSourcesJar task and to the "build" task
|
||||
// if it is present.
|
||||
// If you remove this line, sources will not be generated.
|
||||
withSourcesJar()
|
||||
// Loom will automatically attach sourcesJar to a RemapSourcesJar task and to the "build" task
|
||||
// if it is present.
|
||||
// If you remove this line, sources will not be generated.
|
||||
withSourcesJar()
|
||||
|
||||
sourceCompatibility = JavaVersion.VERSION_21
|
||||
targetCompatibility = JavaVersion.VERSION_21
|
||||
sourceCompatibility = JavaVersion.VERSION_21
|
||||
targetCompatibility = JavaVersion.VERSION_21
|
||||
}
|
||||
|
||||
jar {
|
||||
inputs.property "archivesName", project.base.archivesName
|
||||
inputs.property "archivesName", project.base.archivesName
|
||||
manifest {
|
||||
attributes(Map.of("Main-Class", "io.ib67.sfcraft.bundler.Main"))
|
||||
}
|
||||
from("LICENSE") {
|
||||
rename { "${it}_${inputs.properties.archivesName}" }
|
||||
}
|
||||
}
|
||||
|
||||
from("LICENSE") {
|
||||
rename { "${it}_${inputs.properties.archivesName}"}
|
||||
}
|
||||
remapJar {
|
||||
input = tasks.shadowJar.archiveFile
|
||||
}
|
||||
|
||||
// configure the maven publication
|
||||
publishing {
|
||||
publications {
|
||||
create("mavenJava", MavenPublication) {
|
||||
artifactId = project.archives_base_name
|
||||
from components.java
|
||||
}
|
||||
}
|
||||
publications {
|
||||
create("mavenJava", MavenPublication) {
|
||||
artifactId = project.archives_base_name
|
||||
from components.java
|
||||
}
|
||||
}
|
||||
|
||||
// See https://docs.gradle.org/current/userguide/publishing_maven.html for information on how to set up publishing.
|
||||
repositories {
|
||||
// Add repositories to publish to here.
|
||||
// Notice: This block does NOT have the same function as the block in the top level.
|
||||
// The repositories here will be used for publishing your artifact, not for
|
||||
// retrieving dependencies.
|
||||
}
|
||||
// See https://docs.gradle.org/current/userguide/publishing_maven.html for information on how to set up publishing.
|
||||
repositories {
|
||||
// Add repositories to publish to here.
|
||||
// Notice: This block does NOT have the same function as the block in the top level.
|
||||
// The repositories here will be used for publishing your artifact, not for
|
||||
// retrieving dependencies.
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -1,5 +1,5 @@
|
||||
# Done to increase the memory available to gradle.
|
||||
org.gradle.jvmargs=-Xmx1G
|
||||
org.gradle.jvmargs=-Xmx4G
|
||||
org.gradle.parallel=true
|
||||
|
||||
# Fabric Properties
|
||||
@@ -11,8 +11,8 @@ loom_version=1.11-SNAPSHOT
|
||||
|
||||
# Mod Properties
|
||||
mod_version=1.0.0
|
||||
maven_group=com.example
|
||||
archives_base_name=modid
|
||||
maven_group=io.ib67.sfcraft
|
||||
archives_base_name=justbackup
|
||||
|
||||
# Dependencies
|
||||
fabric_version=0.129.0+1.21.8
|
||||
@@ -3,6 +3,8 @@ package io.ib67.sfcraft;
|
||||
public record Backup(
|
||||
String name,
|
||||
String backupKey,
|
||||
String type,
|
||||
String from,
|
||||
boolean incremental,
|
||||
long sizeTotal
|
||||
) {
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
package io.ib67.sfcraft;
|
||||
|
||||
import com.mojang.brigadier.Command;
|
||||
import com.mojang.brigadier.context.CommandContext;
|
||||
import net.minecraft.server.command.ServerCommandSource;
|
||||
import net.minecraft.text.ClickEvent;
|
||||
import net.minecraft.text.Text;
|
||||
|
||||
import java.awt.*;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
public class BackupCommands {
|
||||
protected final JustBackupMod mod;
|
||||
protected Runnable restoreIssued;
|
||||
|
||||
public BackupCommands(JustBackupMod mod) {
|
||||
this.mod = mod;
|
||||
}
|
||||
|
||||
public int cmdList(CommandContext<ServerCommandSource> ctx) {
|
||||
var s = ctx.getSource();
|
||||
s.sendFeedback(() -> Text.of("Tracked backups:"), false);
|
||||
for (Backup value : mod.tracker.getTrackedBackups().values()) {
|
||||
var restoreClick = Text.literal("[RESTORE]").withColor(Color.GREEN.getRGB())
|
||||
.styled(it -> it.withClickEvent(new ClickEvent.SuggestCommand("/backup restore " + value.backupKey())));
|
||||
var deleteClick = Text.literal("[DELETE]").withColor(Color.RED.getRGB())
|
||||
.styled(it -> it.withClickEvent(new ClickEvent.SuggestCommand("/backup delete " + value.backupKey())));
|
||||
var message = Text.literal(" - " + value.name() + " (" + value.backupKey() + ") " +
|
||||
(value.incremental() ? "incremental" : "full") + " ").append(restoreClick).append(" ").append(deleteClick);
|
||||
s.sendMessage(message);
|
||||
s.sendMessage(Text.of(" Size: " + Math.floorDiv(value.sizeTotal(), 1024 * 1024) + "MiB"));
|
||||
}
|
||||
return Command.SINGLE_SUCCESS;
|
||||
}
|
||||
|
||||
public int cmdDelete(CommandContext<ServerCommandSource> ctx) {
|
||||
var _backup = ctx.getArgument("backupName", String.class);
|
||||
var backup = mod.tracker.getTrackedBackups().get(_backup);
|
||||
mod.tracker.deleteBackup(backup);
|
||||
ctx.getSource().sendMessage(Text.of("Backup " + _backup + " is deleted successfully."));
|
||||
return Command.SINGLE_SUCCESS;
|
||||
}
|
||||
|
||||
public int cmdIssueBackup(CommandContext<ServerCommandSource> ctx) {
|
||||
var s = ctx.getSource();
|
||||
if (JustBackupMod.PERFORMING_BACKUP.get()) {
|
||||
s.sendMessage(Text.of("You can't issue a backup because another backup is in progress."));
|
||||
return Command.SINGLE_SUCCESS;
|
||||
}
|
||||
mod.issueBackup()
|
||||
.whenComplete((backup, error) -> {
|
||||
if (error != null) {
|
||||
s.sendMessage(Text.of("Cannot backup, error: " + error.getMessage()));
|
||||
} else {
|
||||
s.sendMessage(Text.of("Backup created successfully! name: " + backup.name()));
|
||||
}
|
||||
});
|
||||
return Command.SINGLE_SUCCESS;
|
||||
}
|
||||
|
||||
public int cmdSuspend(CommandContext<ServerCommandSource> ctx) {
|
||||
mod.suspend = !mod.suspend;
|
||||
ctx.getSource().sendMessage(Text.of("Backup disabled: " + mod.suspend));
|
||||
return Command.SINGLE_SUCCESS;
|
||||
}
|
||||
|
||||
public int cmdRestore(CommandContext<ServerCommandSource> ctx) {
|
||||
var s = ctx.getSource();
|
||||
var pm = s.getServer().getPlayerManager();
|
||||
var _backup = ctx.getArgument("backupName", String.class);
|
||||
var backup = mod.tracker.getTrackedBackups().get(_backup);
|
||||
if (backup == null) {
|
||||
s.sendMessage(Text.of("Invalid backup " + _backup));
|
||||
return 0;
|
||||
}
|
||||
if (restoreIssued != null) {
|
||||
s.sendMessage(Text.of("Another restoration is in progress. try /backup cancel to cancel it"));
|
||||
return 0;
|
||||
}
|
||||
s.sendMessage(Text.literal("Please restart your server to take changes"));
|
||||
restoreIssued = () -> {
|
||||
System.out.println("Recovering backup... ");
|
||||
try{
|
||||
var toOverride = Path.of(backup.from());
|
||||
var old = Path.of(backup.from()+"_old");
|
||||
Files.deleteIfExists(old);
|
||||
Files.move(toOverride, old);
|
||||
System.out.println("Moved your old save to "+old);
|
||||
mod.tracker.recoverBackup(backup, toOverride);
|
||||
System.out.println("Recover successfully!");
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
};
|
||||
return Command.SINGLE_SUCCESS;
|
||||
}
|
||||
|
||||
public int cmdHelp(CommandContext<ServerCommandSource> serverCommandSourceCommandContext) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
public int cmdCancelRestore(CommandContext<ServerCommandSource> serverCommandSourceCommandContext) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
package io.ib67.sfcraft;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class BackupManager {
|
||||
public List<Backup> getAllBackup() {
|
||||
}
|
||||
|
||||
public void deleteBackup(Backup backup) {
|
||||
}
|
||||
|
||||
public Backup performBackup() {
|
||||
}
|
||||
}
|
||||
@@ -1,27 +1,34 @@
|
||||
package io.ib67.sfcraft;
|
||||
|
||||
import io.ib67.sfcraft.bundler.Bundle;
|
||||
import io.ib67.sfcraft.config.JustBackupConfig;
|
||||
import io.ib67.sfcraft.mixin.LevelStorageAccessor;
|
||||
import io.ib67.sfcraft.strategy.BackupStrategy;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import net.minecraft.server.MinecraftServer;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import static io.ib67.sfcraft.JustBackupMod.PERFORMING_BACKUP;
|
||||
|
||||
@Log4j2
|
||||
public record BackupWorker(
|
||||
MinecraftServer server,
|
||||
BackupManager manager
|
||||
) implements Runnable {
|
||||
BackupStrategy strategy,
|
||||
JustBackupConfig config
|
||||
) implements Supplier<Backup> {
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public void run() {
|
||||
public Backup get() {
|
||||
if (server == null) {
|
||||
log.warn("MinecraftServer is not initialized yet, not doing backup");
|
||||
return;
|
||||
throw new IllegalStateException("MinecraftServer is not initialized yet, not doing backup");
|
||||
}
|
||||
int i = 0;
|
||||
while (!PERFORMING_BACKUP.compareAndSet(false, true)) {
|
||||
@@ -36,8 +43,20 @@ public record BackupWorker(
|
||||
}
|
||||
var session = ((LevelStorageAccessor) server).getSession();
|
||||
var path = session.getDirectory().getRootPath();
|
||||
manager.performBackup();
|
||||
strategy.createBackup(new WorldDir(Path.of(path)));
|
||||
var backupName = "Backup_" + DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(LocalDateTime.now());
|
||||
var backupParentRoot = Path.of(config.temporaryBackupDir());
|
||||
if (Files.notExists(backupParentRoot)) Files.createDirectories(backupParentRoot);
|
||||
var backupFile = backupParentRoot.resolve(backupName + ".jpack");
|
||||
log.info("Creating bundle {}", backupName);
|
||||
//todo diff
|
||||
Path worldDir = Path.of(path);
|
||||
new Bundle(new WorldDir(worldDir))
|
||||
.compress(config.compress())
|
||||
.allowGunzip(config.allowGunzip())
|
||||
.buildBundle(backupFile);
|
||||
var backup = strategy.createBackup(worldDir, backupFile);
|
||||
Files.deleteIfExists(backupFile);
|
||||
return backup;
|
||||
} finally {
|
||||
PERFORMING_BACKUP.set(false);
|
||||
}
|
||||
|
||||
@@ -2,36 +2,58 @@ package io.ib67.sfcraft;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import com.google.gson.reflect.TypeToken;
|
||||
import com.mojang.brigadier.CommandDispatcher;
|
||||
import com.mojang.brigadier.arguments.StringArgumentType;
|
||||
import io.ib67.sfcraft.config.JustBackupConfig;
|
||||
import io.ib67.sfcraft.config.StorageOption;
|
||||
import io.ib67.sfcraft.config.serializer.StorageOptionSerializer;
|
||||
import io.ib67.sfcraft.mixin.LevelStorageAccessor;
|
||||
import io.ib67.sfcraft.mixin.MixinMinecraftServer;
|
||||
import io.ib67.sfcraft.strategy.BackupStrategy;
|
||||
import io.ib67.sfcraft.strategy.BackupTracker;
|
||||
import io.ib67.sfcraft.strategy.LocalBackupStrategy;
|
||||
import io.ib67.sfcraft.strategy.S3BackupStrategy;
|
||||
import lombok.SneakyThrows;
|
||||
import net.fabricmc.api.ModInitializer;
|
||||
|
||||
import net.fabricmc.fabric.api.command.v2.CommandRegistrationCallback;
|
||||
import net.fabricmc.fabric.api.event.lifecycle.v1.ServerLifecycleEvents;
|
||||
import net.minecraft.command.CommandRegistryAccess;
|
||||
import net.minecraft.server.MinecraftServer;
|
||||
import net.minecraft.server.command.CommandManager;
|
||||
import net.minecraft.server.command.ServerCommandSource;
|
||||
import net.minecraft.server.world.ServerWorld;
|
||||
import net.minecraft.text.Text;
|
||||
import org.jetbrains.annotations.UnknownNullability;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import static net.minecraft.server.command.CommandManager.argument;
|
||||
import static net.minecraft.server.command.CommandManager.literal;
|
||||
|
||||
public class JustBackupMod implements ModInitializer {
|
||||
public static final String MOD_ID = "justbackup";
|
||||
public static final Logger LOGGER = LoggerFactory.getLogger(MOD_ID);
|
||||
/**
|
||||
* This state indicates:
|
||||
* 1. A minecraft auto-save is in-progress.
|
||||
* 2. The backup is in-progress
|
||||
*/
|
||||
public static final AtomicBoolean PERFORMING_BACKUP = new AtomicBoolean(false);
|
||||
public static final Gson GSON = new GsonBuilder().setPrettyPrinting()
|
||||
.registerTypeAdapter(StorageOption.class, new StorageOptionSerializer(
|
||||
@@ -41,38 +63,95 @@ public class JustBackupMod implements ModInitializer {
|
||||
.create();
|
||||
protected ScheduledExecutorService scheduledBackupExecutor;
|
||||
protected JustBackupConfig config;
|
||||
protected BackupStrategy strategy;
|
||||
protected BackupTracker tracker;
|
||||
protected Path backupIndexPath;
|
||||
protected volatile MinecraftServer server;
|
||||
protected volatile boolean suspend;
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public void onInitialize() {
|
||||
LOGGER.info("Loading backup configuration");
|
||||
backupIndexPath = Path.of("config").resolve("just_backup_index.json");
|
||||
var configPath = Path.of("config").resolve("just_backup.json");
|
||||
if (Files.notExists(configPath.getParent())) {
|
||||
if (Files.notExists(configPath.getParent()))
|
||||
Files.createDirectories(configPath.getParent());
|
||||
}
|
||||
if (Files.notExists(configPath)) {
|
||||
if (Files.notExists(configPath))
|
||||
saveConfig(configPath);
|
||||
}
|
||||
if (Files.notExists(backupIndexPath))
|
||||
Files.createFile(backupIndexPath);
|
||||
|
||||
config = GSON.fromJson(Files.readString(configPath), JustBackupConfig.class);
|
||||
this.strategy = createStrategy();
|
||||
var strategy = createStrategy();
|
||||
this.tracker = new BackupTracker(strategy, GSON.fromJson(Files.readString(backupIndexPath), new TypeToken<>() {
|
||||
}));
|
||||
LOGGER.info("Configuration has been loaded. Using storage option: {}", config.option().type());
|
||||
scheduledBackupExecutor = Executors.newSingleThreadScheduledExecutor();
|
||||
registerHandlers();
|
||||
}
|
||||
|
||||
private void registerHandlers() {
|
||||
ServerLifecycleEvents.SERVER_STARTED.register(server -> {
|
||||
LOGGER.info("Launching background threads...");
|
||||
this.server = server;
|
||||
scheduledBackupExecutor.scheduleAtFixedRate(new BackupWorker(server, strategy),
|
||||
scheduledBackupExecutor.scheduleAtFixedRate(this::issueBackup,
|
||||
config.backupIntervalMinutes(), config.backupIntervalMinutes(), TimeUnit.MINUTES);
|
||||
});
|
||||
ServerLifecycleEvents.SERVER_STOPPED.register(server -> {
|
||||
try {
|
||||
Files.deleteIfExists(Path.of(config.temporaryBackupDir()));
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
CommandRegistrationCallback.EVENT.register(this::registerCommand);
|
||||
}
|
||||
|
||||
public CompletableFuture<Backup> issueBackup() {
|
||||
if (suspend) {
|
||||
return CompletableFuture.failedFuture(new IllegalStateException("Backup is temporarily disabled"));
|
||||
}
|
||||
server.getPlayerManager().broadcast(Text.literal("The server will begin the backup shortly, you may experience some lag."), false);
|
||||
var worker = new BackupWorker(server, tracker, config);
|
||||
return CompletableFuture.supplyAsync(worker, scheduledBackupExecutor).whenComplete(this::onBackupComplete);
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
private void onBackupComplete(@UnknownNullability Backup backup, @UnknownNullability Throwable throwable) {
|
||||
if (throwable != null) {
|
||||
server.getPlayerManager().broadcast(Text.literal("Backup failed. For administrators, please check your server console."), false);
|
||||
throwable.printStackTrace();
|
||||
} else {
|
||||
Files.writeString(backupIndexPath, GSON.toJson(tracker.getTrackedBackups()));
|
||||
server.getPlayerManager().broadcast(Text.literal("Backup success. " + backup), false);
|
||||
}
|
||||
}
|
||||
|
||||
private void registerCommand(
|
||||
CommandDispatcher<ServerCommandSource> dispatcher,
|
||||
CommandRegistryAccess registry, CommandManager.RegistrationEnvironment env) {
|
||||
var commandHandler = new BackupCommands(this);
|
||||
dispatcher.register(literal("backup")
|
||||
.requires(it -> it.hasPermissionLevel(3))
|
||||
.then(literal("help").executes(commandHandler::cmdHelp))
|
||||
.then(literal("list").executes(commandHandler::cmdList))
|
||||
.then(literal("cancelrestore").executes(commandHandler::cmdCancelRestore))
|
||||
.then(literal("delete")
|
||||
.then(argument("backupName", StringArgumentType.greedyString())
|
||||
.executes(commandHandler::cmdDelete)))
|
||||
.then(literal("restore")
|
||||
.then(argument("backupName", StringArgumentType.greedyString())
|
||||
.executes(commandHandler::cmdRestore)))
|
||||
.then(literal("create").executes(commandHandler::cmdIssueBackup))
|
||||
.then(literal("suspend").executes(commandHandler::cmdSuspend))
|
||||
);
|
||||
}
|
||||
|
||||
private BackupStrategy createStrategy() {
|
||||
switch (config.option()) {
|
||||
case StorageOption.Local local -> {
|
||||
|
||||
}
|
||||
}
|
||||
return switch (config.option()) {
|
||||
case StorageOption.Local local -> new LocalBackupStrategy(local);
|
||||
case StorageOption.S3 s3 -> new S3BackupStrategy(s3);
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -83,8 +162,14 @@ public class JustBackupMod implements ModInitializer {
|
||||
60,
|
||||
5,
|
||||
true,
|
||||
new StorageOption.Local("backups")
|
||||
true,
|
||||
"./.backup_cache",
|
||||
new StorageOption.Local("backups", 0)
|
||||
);
|
||||
var path = Path.of(cfg.temporaryBackupDir());
|
||||
if (Files.notExists(path)) {
|
||||
Files.createDirectories(path);
|
||||
}
|
||||
Files.writeString(configPath, GSON.toJson(cfg));
|
||||
}
|
||||
}
|
||||
@@ -42,7 +42,9 @@ public record WorldDir(
|
||||
public List<Path> otherFiles() {
|
||||
var otherDimensions = otherDimensions();
|
||||
try (var f = Files.walk(root)) {
|
||||
return f.filter(it -> otherDimensions.stream().noneMatch(a -> it.startsWith(a.root)))
|
||||
return f.filter(it -> Files.isRegularFile(it)
|
||||
&& otherDimensions.stream().noneMatch(a -> it.startsWith(a.root)))
|
||||
.filter(it -> !it.getFileName().toString().contains("session.lock"))
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
@@ -50,7 +52,9 @@ public record WorldDir(
|
||||
@SneakyThrows
|
||||
public List<Path> everything() {
|
||||
try (var f = Files.walk(root)) {
|
||||
return f.toList();
|
||||
return f.filter(Files::isRegularFile)
|
||||
.filter(it -> !it.getFileName().toString().contains("session.lock"))
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
|
||||
import io.ib67.sfcraft.WorldDir;
|
||||
import io.netty.buffer.Unpooled;
|
||||
import lombok.Data;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.experimental.Accessors;
|
||||
import org.apache.commons.lang3.Validate;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
@Accessors(fluent = true)
|
||||
@Data
|
||||
public class Bundle {
|
||||
public static final int MAGIC = 0xBACEBACE;
|
||||
public static final int COMPRESSED_MAGIC = 0xBACEAAAA;
|
||||
protected final WorldDir dir;
|
||||
protected boolean compress = true;
|
||||
protected boolean allowGunzip = true;
|
||||
protected boolean allowParallel = false;
|
||||
|
||||
|
||||
public Bundle(WorldDir dir) {
|
||||
this.dir = dir;
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
public static void unbundleFile(Path bundle, Path out) {
|
||||
try (var bundlerReader = new SerialBundlerReader(bundle)) {
|
||||
while (bundlerReader.hasEntry()) {
|
||||
var entry = bundlerReader.readEntry();
|
||||
var target = out.resolve(entry.fileName());
|
||||
Files.createDirectories(target.getParent());
|
||||
bundlerReader.writeTo(FileChannel.open(target, StandardOpenOption.WRITE,
|
||||
StandardOpenOption.CREATE_NEW), entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
public void buildBundle(Path backupFile) {
|
||||
checkCondition();
|
||||
var sortedEntries = dir.everything().stream().sorted((a, b) -> {
|
||||
boolean aMatch = a.endsWith(".mca") || a.endsWith(".dat");
|
||||
boolean bMatch = b.endsWith(".mca") || b.endsWith(".dat");
|
||||
|
||||
if (aMatch && !bMatch) return -1;
|
||||
if (!aMatch && bMatch) return 1;
|
||||
return a.compareTo(b);
|
||||
}).map(dir.root()::relativize).toList();
|
||||
if (Files.notExists(backupFile.getParent()))
|
||||
Files.createDirectories(backupFile.getParent());
|
||||
SerialBundlerWriter writer;
|
||||
if (compress) {
|
||||
var fout = Files.newOutputStream(backupFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
|
||||
writeMagic(fout, null, true);
|
||||
var out = new ZstdOutputStreamNoFinalizer(fout);
|
||||
writer = new SerialBundlerWriter(out);
|
||||
} else {
|
||||
var chan = FileChannel.open(backupFile);
|
||||
writer = new SerialBundlerWriter(chan, 0);
|
||||
writeMagic(null, chan, false);
|
||||
}
|
||||
// todo parallelism
|
||||
writeEntries(sortedEntries, writer, allowGunzip && compress);
|
||||
writer.close();
|
||||
|
||||
try (var backup = FileChannel.open(backupFile, StandardOpenOption.WRITE, StandardOpenOption.APPEND);
|
||||
var bos = new ByteArrayOutputStream();
|
||||
var metadata = new EntryOutputStream(new ZstdOutputStreamNoFinalizer(bos))) {
|
||||
var entries = writer.getEntries();
|
||||
for (BundlerEntry entry : entries) {
|
||||
metadata.write(entry);
|
||||
}
|
||||
metadata.flush();
|
||||
var result = bos.toByteArray();
|
||||
var buffer = ByteBuffer.allocate(result.length +4 + 4);
|
||||
buffer.put(result);
|
||||
buffer.putInt(entries.size());
|
||||
buffer.putInt(result.length);
|
||||
writeFully(backup, buffer);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static void writeMagic(OutputStream out, FileChannel channel, boolean compress) throws IOException {
|
||||
var magic = compress ? Bundle.COMPRESSED_MAGIC : Bundle.MAGIC;
|
||||
if (channel != null) {
|
||||
channel.write(ByteBuffer.allocate(4).putInt(magic).flip());
|
||||
} else {
|
||||
out.write(ByteBuffer.allocate(4).putInt(magic).array());
|
||||
}
|
||||
}
|
||||
|
||||
private void writeFully(FileChannel chan, ByteBuffer buf) throws IOException {
|
||||
buf.flip();
|
||||
while (buf.hasRemaining()) {
|
||||
chan.write(buf);
|
||||
}
|
||||
}
|
||||
|
||||
private void checkCondition() {
|
||||
if (allowParallel) {
|
||||
Validate.isTrue(!compress && !allowGunzip);
|
||||
}
|
||||
}
|
||||
|
||||
private void writeEntries(
|
||||
List<Path> sortedEntries,
|
||||
SerialBundlerWriter writer,
|
||||
boolean allowGunzip) throws IOException {
|
||||
for (Path sortedEntry : sortedEntries) {
|
||||
var fileName = sortedEntry.getFileName();
|
||||
var shouldGunzip = allowGunzip && (fileName.endsWith("mca") || fileName.endsWith("dat"));
|
||||
if (shouldGunzip) {
|
||||
writer.writeGunzip(dir.root().resolve(sortedEntry), sortedEntry.toString());
|
||||
} else {
|
||||
writer.writePlain(dir.root().resolve(sortedEntry), sortedEntry.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -6,5 +6,4 @@ public record BundlerEntry(
|
||||
long fileLength,
|
||||
long offset
|
||||
) {
|
||||
public static final int MAGIC = 0xBACEBACE;
|
||||
}
|
||||
@@ -24,7 +24,7 @@ public class EntryInputStream extends FilterInputStream {
|
||||
var gunzipped = len < 0;
|
||||
len = Math.abs(len);
|
||||
var name = new String(readNBytes(len)).intern();
|
||||
var buf = ByteBuffer.wrap(readNBytes(8));
|
||||
var buf = ByteBuffer.wrap(readNBytes(8 + 8));
|
||||
var fileLen = buf.getLong();
|
||||
var fileOffset = buf.getLong();
|
||||
return new BundlerEntry(name, gunzipped, fileLen, fileOffset);
|
||||
|
||||
@@ -20,10 +20,6 @@ public class EntryOutputStream extends FilterOutputStream {
|
||||
super(out);
|
||||
}
|
||||
|
||||
public void writeMagic() throws IOException {
|
||||
out.write(BundlerEntry.MAGIC);
|
||||
}
|
||||
|
||||
public void write(BundlerEntry entry) throws IOException {
|
||||
var fileName = entry.fileName().getBytes(StandardCharsets.UTF_8);
|
||||
var buf = ByteBuffer.allocate(4).putInt(entry.unGzipped() ? fileName.length * -1 : fileName.length);
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
import io.ib67.kiwi.ArgOpts;
|
||||
import io.ib67.sfcraft.WorldDir;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) {
|
||||
var opts = ArgOpts.builder()
|
||||
.args(args).description("Tools for .jpack format bundle").programName("jpack").build();
|
||||
var in = opts.string("in", "Input. Can be directory or jpack bundle", null);
|
||||
var out = opts.string("out", "Output. Can be jpack bundle or directory", null);
|
||||
var help = opts.bool("help", false);
|
||||
var allowGunzip = opts.bool("gunzip",
|
||||
"Should we recompress NBT files for maximum compression rate (depends on compress)", true);
|
||||
var compress = opts.bool("compress", "Should we compress?", true);
|
||||
var parallel = opts.bool("parallel", "Use parallelized bundler. This conflicts with compress options", false);
|
||||
if (help) {
|
||||
opts.printHelp(System.out);
|
||||
return;
|
||||
}
|
||||
if (allowGunzip && !compress) {
|
||||
System.out.println("allowGunzip depends on compress feature.");
|
||||
return;
|
||||
}
|
||||
if (in == null || out == null) {
|
||||
System.out.println("Both in and out should be provided");
|
||||
return;
|
||||
}
|
||||
var inPath = Path.of(in);
|
||||
var outPath = Path.of(out);
|
||||
if (!Files.exists(inPath)) {
|
||||
System.out.println("The input isn't exists.");
|
||||
return;
|
||||
}
|
||||
if (Files.exists(outPath)) {
|
||||
System.out.println("The output is already exists. Please move it to other places");
|
||||
return;
|
||||
}
|
||||
if (in.endsWith(".jpack")) {
|
||||
Bundle.unbundleFile(inPath, outPath);
|
||||
} else if (out.endsWith(".jpack")) {
|
||||
if (!Files.isDirectory(inPath))
|
||||
throw new IllegalArgumentException("Your input should be a world directory.");
|
||||
new Bundle(new WorldDir(inPath))
|
||||
.allowGunzip(allowGunzip)
|
||||
.compress(compress)
|
||||
.allowParallel(parallel)
|
||||
.buildBundle(outPath);
|
||||
} else {
|
||||
System.out.println("Invalid arguments. You should provide a pair of jpack and directory to act as in/output.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -19,31 +19,37 @@ public class SerialBundlerReader implements Closeable {
|
||||
protected final EntryInputStream metadataIn;
|
||||
protected final byte[] buffer = new byte[4096];
|
||||
protected boolean entryRead;
|
||||
protected int entries;
|
||||
|
||||
public SerialBundlerReader(InputStream in, InputStream metadataIn) {
|
||||
this.in = Objects.requireNonNull(in);
|
||||
this.metadataIn = new EntryInputStream(Objects.requireNonNull(metadataIn));
|
||||
inChannel = null;
|
||||
}
|
||||
// public SerialBundlerReader(InputStream in, InputStream metadataIn) {
|
||||
// this.in = Objects.requireNonNull(in);
|
||||
// this.metadataIn = new EntryInputStream(Objects.requireNonNull(metadataIn));
|
||||
// inChannel = null;
|
||||
// }
|
||||
|
||||
public SerialBundlerReader(Path path) throws IOException {
|
||||
var raf = new RandomAccessFile(path.toFile(), "r");
|
||||
var totalLen = Files.size(path);
|
||||
var magic = raf.readInt();
|
||||
var compressed = magic == Bundle.COMPRESSED_MAGIC;
|
||||
if (!compressed && magic != Bundle.MAGIC) throw new IOException("Invalid bundle file");
|
||||
|
||||
// read length
|
||||
raf.seek(totalLen - 4);
|
||||
var metadataLength = raf.readInt();
|
||||
var metadataBegin = totalLen - 4 - metadataLength;
|
||||
raf.seek(totalLen - 8);
|
||||
entries = raf.readInt();
|
||||
// read metadata
|
||||
var metadataBegin = totalLen - 8 - metadataLength;
|
||||
var metadataBytes = new byte[metadataLength];
|
||||
raf.seek(metadataBegin);
|
||||
raf.readFully(metadataBytes);
|
||||
|
||||
this.metadataIn = new EntryInputStream(new ZstdInputStreamNoFinalizer(new ByteArrayInputStream(metadataBytes)));
|
||||
if (ByteBuffer.wrap(metadataIn.readNBytes(4)).getInt() != BundlerEntry.MAGIC) {
|
||||
throw new IllegalArgumentException("Invalid Bundler entry");
|
||||
}
|
||||
// check compressed
|
||||
raf.seek(0);
|
||||
raf.seek(4);
|
||||
var fis = Channels.newInputStream(raf.getChannel());
|
||||
if (raf.readInt() == Zstd.magicNumber()) {
|
||||
if (compressed) {
|
||||
in = new ZstdInputStreamNoFinalizer(fis);
|
||||
inChannel = null;
|
||||
} else {
|
||||
@@ -54,12 +60,14 @@ public class SerialBundlerReader implements Closeable {
|
||||
|
||||
@SneakyThrows
|
||||
public boolean hasEntry() {
|
||||
return metadataIn.available() > 0;
|
||||
return entries > 0;
|
||||
}
|
||||
|
||||
public BundlerEntry readEntry() throws IOException {
|
||||
if (!hasEntry()) throw new IOException("All entries are read");
|
||||
var entry = metadataIn.readEntry();
|
||||
entryRead = true;
|
||||
entries--;
|
||||
return entry;
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import java.nio.channels.FileChannel;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
@@ -28,12 +29,14 @@ public class SerialBundlerWriter implements Closeable {
|
||||
public SerialBundlerWriter(OutputStream out) {
|
||||
Objects.requireNonNull(out);
|
||||
this.out = out;
|
||||
this.entries = new ArrayList<>();
|
||||
channel = null;
|
||||
}
|
||||
|
||||
public SerialBundlerWriter(FileChannel channel, long offset) {
|
||||
Objects.requireNonNull(channel);
|
||||
this.offset = offset;
|
||||
this.entries = new ArrayList<>();
|
||||
this.channel = channel;
|
||||
out = null;
|
||||
}
|
||||
@@ -50,10 +53,12 @@ public class SerialBundlerWriter implements Closeable {
|
||||
public void writePlain(Path path, String fileName) throws IOException {
|
||||
if (channel != null) {
|
||||
writeZC(path, fileName);
|
||||
return;
|
||||
}
|
||||
try (var in = Files.newInputStream(path)) {
|
||||
in.transferTo(out);
|
||||
} else {
|
||||
var current = offset;
|
||||
try (var in = Files.newInputStream(path)) {
|
||||
in.transferTo(out);
|
||||
}
|
||||
entries.add(new BundlerEntry(fileName, false, Files.size(path), current));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,8 +7,9 @@ import java.util.List;
|
||||
public record JustBackupConfig(
|
||||
long backupIntervalMinutes,
|
||||
int keepBackups,
|
||||
boolean allowParallel,
|
||||
boolean allowGunzip,
|
||||
boolean compress,
|
||||
String temporaryBackupDir,
|
||||
StorageOption option
|
||||
) {
|
||||
public JustBackupConfig {
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package io.ib67.sfcraft.config;
|
||||
|
||||
import org.apache.commons.lang3.Validate;
|
||||
|
||||
public sealed interface StorageOption {
|
||||
String type();
|
||||
|
||||
@@ -8,20 +10,32 @@ public sealed interface StorageOption {
|
||||
String accessKey,
|
||||
String secretKey,
|
||||
String endpoint,
|
||||
String bucket
|
||||
String bucket,
|
||||
String prefix,
|
||||
boolean enforcePathStyle,
|
||||
int retryAmount
|
||||
) implements StorageOption {
|
||||
@Override
|
||||
public String type() {
|
||||
return "s3";
|
||||
}
|
||||
|
||||
public S3 {
|
||||
Validate.isTrue(retryAmount >= 0);
|
||||
}
|
||||
}
|
||||
|
||||
record Local(
|
||||
String saveDir
|
||||
String saveDir,
|
||||
long diskSizeQuotaBytes
|
||||
) implements StorageOption {
|
||||
@Override
|
||||
public String type() {
|
||||
return "local";
|
||||
}
|
||||
|
||||
public Local {
|
||||
Validate.isTrue(diskSizeQuotaBytes >= 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,16 +2,20 @@ package io.ib67.sfcraft.strategy;
|
||||
|
||||
import io.ib67.sfcraft.Backup;
|
||||
import io.ib67.sfcraft.WorldDir;
|
||||
import io.ib67.sfcraft.bundler.Bundle;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
public interface BackupStrategy {
|
||||
/**
|
||||
* create a backup
|
||||
* @param dir world to be bundled
|
||||
* @return a key to this backup
|
||||
*/
|
||||
Backup createBackup(WorldDir dir, boolean compress);
|
||||
Backup createBackup(Path source, Path bundle);
|
||||
|
||||
void recoverBackup(Backup backup, Path restorePath);
|
||||
|
||||
void deleteBackup(Backup backup);
|
||||
|
||||
boolean isAvailable();
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package io.ib67.sfcraft.strategy;
|
||||
|
||||
import io.ib67.sfcraft.Backup;
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
public class BackupTracker implements BackupStrategy {
|
||||
protected final BackupStrategy underlying;
|
||||
@Getter
|
||||
protected final Map<String, Backup> trackedBackups;
|
||||
|
||||
public BackupTracker(BackupStrategy underlying, Map<String, Backup> trackedBackups) {
|
||||
this.underlying = Objects.requireNonNull(underlying);
|
||||
this.trackedBackups = trackedBackups == null ? new HashMap<>() : trackedBackups;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Backup createBackup(Path from, Path pathToBundle) {
|
||||
var result = underlying.createBackup(from, pathToBundle);
|
||||
trackedBackups.put(result.backupKey(), result);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void recoverBackup(Backup backup, Path restorePath) {
|
||||
underlying.recoverBackup(backup, restorePath);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteBackup(Backup backup) {
|
||||
underlying.deleteBackup(backup);
|
||||
trackedBackups.remove(backup.backupKey());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isAvailable() {
|
||||
return underlying.isAvailable();
|
||||
}
|
||||
}
|
||||
@@ -4,14 +4,12 @@ import com.github.luben.zstd.Zstd;
|
||||
import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
|
||||
import io.ib67.sfcraft.Backup;
|
||||
import io.ib67.sfcraft.WorldDir;
|
||||
import io.ib67.sfcraft.bundler.BundlerEntry;
|
||||
import io.ib67.sfcraft.bundler.EntryOutputStream;
|
||||
import io.ib67.sfcraft.bundler.SerialBundlerReader;
|
||||
import io.ib67.sfcraft.bundler.SerialBundlerWriter;
|
||||
import io.ib67.sfcraft.bundler.*;
|
||||
import io.ib67.sfcraft.config.StorageOption;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.apache.commons.lang3.Validate;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
@@ -25,80 +23,61 @@ import java.time.format.DateTimeFormatter;
|
||||
import java.util.*;
|
||||
|
||||
@Log4j2
|
||||
@RequiredArgsConstructor
|
||||
public class LocalBackupStrategy implements BackupStrategy {
|
||||
protected final StorageOption.Local option;
|
||||
protected final Path backupParentRoot = Path.of(option.saveDir());
|
||||
protected final boolean allowGunzip;
|
||||
protected final Path backupParentRoot;
|
||||
|
||||
@SneakyThrows
|
||||
public LocalBackupStrategy(StorageOption.Local option) {
|
||||
this.option = option;
|
||||
backupParentRoot = Path.of(option.saveDir());
|
||||
Files.createDirectories(backupParentRoot);
|
||||
}
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public Backup createBackup(WorldDir dir, boolean compress) {
|
||||
var sortedEntries = dir.everything().stream().sorted((a, b) -> {
|
||||
boolean aMatch = a.endsWith(".mca") || a.endsWith(".dat");
|
||||
boolean bMatch = b.endsWith(".mca") || b.endsWith(".dat");
|
||||
|
||||
if (aMatch && !bMatch) return -1;
|
||||
if (!aMatch && bMatch) return 1;
|
||||
return a.compareTo(b);
|
||||
}).map(dir.root()::relativize).toList();
|
||||
var backupName = "Backup_" + DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(LocalDateTime.now());
|
||||
var backupFile = backupParentRoot.resolve(backupName + ".jbp");
|
||||
SerialBundlerWriter writer;
|
||||
if (compress) {
|
||||
var out = new ZstdOutputStreamNoFinalizer(Files.newOutputStream(backupFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE));
|
||||
writer = new SerialBundlerWriter(out);
|
||||
} else {
|
||||
var chan = FileChannel.open(backupFile);
|
||||
writer = new SerialBundlerWriter(chan, 0);
|
||||
}
|
||||
writeEntries(sortedEntries, writer, allowGunzip && compress);
|
||||
writer.close();
|
||||
|
||||
try (var backup = FileChannel.open(backupFile);
|
||||
var bos = new ByteArrayOutputStream();
|
||||
var metadata = new EntryOutputStream(new ZstdOutputStreamNoFinalizer(bos))) {
|
||||
metadata.writeMagic();
|
||||
for (BundlerEntry entry : writer.getEntries()) {
|
||||
metadata.write(entry);
|
||||
}
|
||||
var result = bos.toByteArray();
|
||||
var buffer = ByteBuffer.allocate(result.length + 4);
|
||||
buffer.put(result);
|
||||
buffer.putLong(result.length);
|
||||
backup.write(buffer);
|
||||
}
|
||||
return new Backup(backupName, backupFile.getFileName().toString(), false, Files.size(backupFile));
|
||||
public Backup createBackup(Path from, Path pathToBundle) {
|
||||
var result = new Backup(
|
||||
pathToBundle.getFileName().toString(),
|
||||
pathToBundle.getFileName().toString(),
|
||||
"local", from.toString(), false, Files.size(pathToBundle));
|
||||
Files.move(pathToBundle, backupParentRoot.resolve(pathToBundle.getFileName()));
|
||||
return result;
|
||||
}
|
||||
|
||||
private void writeEntries(
|
||||
List<Path> sortedEntries,
|
||||
SerialBundlerWriter writer,
|
||||
boolean allowGunzip) throws IOException {
|
||||
for (Path sortedEntry : sortedEntries) {
|
||||
var fileName = sortedEntry.getFileName();
|
||||
var shouldGunzip = allowGunzip && (fileName.endsWith("mca") || fileName.endsWith("dat"));
|
||||
if (shouldGunzip) {
|
||||
writer.writeGunzip(sortedEntry, sortedEntry.toString());
|
||||
} else {
|
||||
writer.writePlain(sortedEntry, sortedEntry.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public void recoverBackup(Backup backup, Path restorePath) {
|
||||
// check if it is compressed
|
||||
var backupFile = backupParentRoot.resolve(backup.backupKey());
|
||||
if (Files.notExists(backupFile)) throw new IllegalArgumentException("Backup " + backupFile + " does not exist");
|
||||
try (var bundlerReader = new SerialBundlerReader(backupFile)) {
|
||||
while (bundlerReader.hasEntry()) {
|
||||
var entry = bundlerReader.readEntry();
|
||||
var target = restorePath.resolve(entry.fileName());
|
||||
bundlerReader.writeTo(FileChannel.open(target), entry);
|
||||
Bundle.unbundleFile(backupFile, restorePath);
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
@Override
|
||||
public void deleteBackup(Backup backup) {
|
||||
Validate.isTrue("local".equals(backup.backupKey()));
|
||||
var bundlePath = backupParentRoot.resolve(backup.backupKey());
|
||||
Files.deleteIfExists(bundlePath);
|
||||
}
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public boolean isAvailable() {
|
||||
if (option.diskSizeQuotaBytes() == 0) return true;
|
||||
try (var f = Files.walk(backupParentRoot)) {
|
||||
if (f.mapToLong(it->{
|
||||
try {
|
||||
return Files.size(it);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}).sum() >= option.diskSizeQuotaBytes()) {
|
||||
log.warn("Have no space to set another backup! ");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package io.ib67.sfcraft.strategy;
|
||||
|
||||
import io.ib67.sfcraft.Backup;
|
||||
import io.ib67.sfcraft.WorldDir;
|
||||
import io.ib67.sfcraft.bundler.Bundle;
|
||||
import io.ib67.sfcraft.config.StorageOption;
|
||||
import io.minio.BucketExistsArgs;
|
||||
import io.minio.MinioClient;
|
||||
import io.minio.RemoveObjectArgs;
|
||||
import io.minio.UploadObjectArgs;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URI;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
@Log4j2
|
||||
public class S3BackupStrategy implements BackupStrategy {
|
||||
protected final StorageOption.S3 option;
|
||||
protected final MinioClient minio;
|
||||
protected final AtomicBoolean uploading = new AtomicBoolean(false);
|
||||
|
||||
@SneakyThrows
|
||||
public S3BackupStrategy(StorageOption.S3 option) {
|
||||
this.option = option;
|
||||
this.minio = MinioClient.builder()
|
||||
.endpoint(URI.create(option.endpoint()).toURL())
|
||||
.credentials(option.accessKey(), option.secretKey())
|
||||
.region(option.region())
|
||||
.build();
|
||||
if (option.enforcePathStyle()) {
|
||||
minio.disableVirtualStyleEndpoint();
|
||||
} else {
|
||||
minio.enableVirtualStyleEndpoint();
|
||||
}
|
||||
log.info("Testing S3 credentials");
|
||||
var exists = minio.bucketExists(BucketExistsArgs.builder().bucket(option.bucket()).build());
|
||||
if (!exists) throw new IllegalStateException("Bucket '" + option.bucket() + "' does not exist");
|
||||
}
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public Backup createBackup(Path from, Path pathToBundle) {
|
||||
if (!uploading.compareAndSet(false, true)) {
|
||||
throw new IllegalStateException("Last upload is not done yet! keep waiting...");
|
||||
}
|
||||
int i = 0;
|
||||
do {
|
||||
try {
|
||||
var object = option.prefix() + "/" + pathToBundle.getFileName().toString();
|
||||
minio.uploadObject(UploadObjectArgs.builder()
|
||||
.contentType("application/octet-stream")
|
||||
.object(object)
|
||||
.filename(pathToBundle.toString())
|
||||
.build());
|
||||
//todo also calculate sha?
|
||||
return new Backup(pathToBundle.getFileName().toString(),
|
||||
object, "s3",
|
||||
from.toString(),
|
||||
false,
|
||||
Files.size(pathToBundle));
|
||||
} catch (IOException e) {
|
||||
if (i >= option.retryAmount()) {
|
||||
throw new IllegalStateException("Cannot upload " + pathToBundle + " to S3");
|
||||
}
|
||||
var nextTry = Math.pow(2, i++);
|
||||
log.error("Error uploading S3. trying in {}s later. ({}/{})", nextTry, i, option.retryAmount(), e);
|
||||
Thread.sleep((long) (nextTry * 1000));
|
||||
} catch (Exception e) {
|
||||
throw new IllegalStateException("Unrecoverable exception occurred when uploading " + pathToBundle, e);
|
||||
} finally {
|
||||
uploading.set(false);
|
||||
}
|
||||
} while (true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void recoverBackup(Backup backup, Path restorePath) {
|
||||
//todo http range
|
||||
}
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public void deleteBackup(Backup backup) {
|
||||
minio.removeObject(RemoveObjectArgs.builder()
|
||||
.bucket(backup.backupKey()).build());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isAvailable() {
|
||||
if (uploading.get()) {
|
||||
log.warn("Last S3 Backup is still uploading!");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user