Refactor backup system and migrate from submodule to shadow dependencies
- Remove s3-lite submodule and replace with AWS SDK and shadow plugin - Implement comprehensive backup commands with list, delete, create, restore, and suspend functionality - Add bundle format with compression support using Zstd and gunzip optimization - Implement local and S3 backup strategies with configurable options - Add backup tracking system with JSON persistence - Improve world directory handling and file exclusion logic - Add standalone bundler tool for bundle creation/extraction - Update Gradle configuration with shadow jar support and dependency management - Enhance configuration system with additional options for compression and temporary directories
This commit is contained in:
1 parent
6a2642ab93
commit
fb42be8155
21 files changed
+746
-183
No files matched your search
+71
-53
@@ -1,94 +1,112 @@
|
|||||||
plugins {
|
plugins {
|
||||||
id 'fabric-loom' version "${loom_version}"
|
id 'fabric-loom' version "${loom_version}"
|
||||||
id 'maven-publish'
|
id 'maven-publish'
|
||||||
id("io.freefair.lombok") version "9.0.0-rc2"
|
id("io.freefair.lombok") version "9.0.0-rc2"
|
||||||
|
id("com.gradleup.shadow") version '9.1.0'
|
||||||
}
|
}
|
||||||
|
|
||||||
version = project.mod_version
|
version = project.mod_version
|
||||||
group = project.maven_group
|
group = project.maven_group
|
||||||
|
|
||||||
base {
|
base {
|
||||||
archivesName = project.archives_base_name
|
archivesName = project.archives_base_name
|
||||||
}
|
}
|
||||||
|
|
||||||
repositories {
|
repositories {
|
||||||
// Add repositories to retrieve artifacts from in here.
|
// Add repositories to retrieve artifacts from in here.
|
||||||
// You should only use this when depending on other mods because
|
// You should only use this when depending on other mods because
|
||||||
// Loom adds the essential maven repositories to download Minecraft and libraries from automatically.
|
// Loom adds the essential maven repositories to download Minecraft and libraries from automatically.
|
||||||
// See https://docs.gradle.org/current/userguide/declaring_repositories.html
|
// See https://docs.gradle.org/current/userguide/declaring_repositories.html
|
||||||
// for more information about repositories.
|
// for more information about repositories.
|
||||||
|
maven {
|
||||||
|
name = "Kalculos"
|
||||||
|
url = "https://repo.sfclub.cc/snapshots"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
loom {
|
loom {
|
||||||
splitEnvironmentSourceSets()
|
splitEnvironmentSourceSets()
|
||||||
|
|
||||||
mods {
|
mods {
|
||||||
"modid" {
|
"justbackup" {
|
||||||
sourceSet sourceSets.main
|
sourceSet sourceSets.main
|
||||||
sourceSet sourceSets.client
|
sourceSet sourceSets.client
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
dependencies {
|
dependencies {
|
||||||
// To change the versions see the gradle.properties file
|
// To change the versions see the gradle.properties file
|
||||||
minecraft "com.mojang:minecraft:${project.minecraft_version}"
|
minecraft "com.mojang:minecraft:${project.minecraft_version}"
|
||||||
mappings "net.fabricmc:yarn:${project.yarn_mappings}:v2"
|
mappings "net.fabricmc:yarn:${project.yarn_mappings}:v2"
|
||||||
modImplementation "net.fabricmc:fabric-loader:${project.loader_version}"
|
modImplementation "net.fabricmc:fabric-loader:${project.loader_version}"
|
||||||
|
|
||||||
// Fabric API. This is technically optional, but you probably want it anyway.
|
// Fabric API. This is technically optional, but you probably want it anyway.
|
||||||
modImplementation "net.fabricmc.fabric-api:fabric-api:${project.fabric_version}"
|
modImplementation "net.fabricmc.fabric-api:fabric-api:${project.fabric_version}"
|
||||||
implementation("io.minio:minio:8.5.17")
|
// https://mvnrepository.com/artifact/software.amazon.awssdk/s3
|
||||||
implementation 'com.github.luben:zstd-jni:1.5.7-4'
|
shadow 'software.amazon.awssdk:s3:2.34.0'
|
||||||
|
shadow("io.ib67.kiwi:lang:1.2.1-SNAPSHOT")
|
||||||
|
shadow 'com.github.luben:zstd-jni:1.5.7-4'
|
||||||
|
}
|
||||||
|
|
||||||
|
configurations.implementation.extendsFrom(configurations.shadow)
|
||||||
|
shadowJar {
|
||||||
|
configurations = [project.configurations.shadow]
|
||||||
|
exclude("META-INF")
|
||||||
|
//relocate 'com.google', 'sfcraft.relocation.google'
|
||||||
}
|
}
|
||||||
|
|
||||||
processResources {
|
processResources {
|
||||||
inputs.property "version", project.version
|
inputs.property "version", project.version
|
||||||
|
|
||||||
filesMatching("fabric.mod.json") {
|
filesMatching("fabric.mod.json") {
|
||||||
expand "version": inputs.properties.version
|
expand "version": inputs.properties.version
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
tasks.withType(JavaCompile).configureEach {
|
tasks.withType(JavaCompile).configureEach {
|
||||||
it.options.release = 21
|
it.options.release = 21
|
||||||
}
|
}
|
||||||
|
|
||||||
java {
|
java {
|
||||||
// Loom will automatically attach sourcesJar to a RemapSourcesJar task and to the "build" task
|
// Loom will automatically attach sourcesJar to a RemapSourcesJar task and to the "build" task
|
||||||
// if it is present.
|
// if it is present.
|
||||||
// If you remove this line, sources will not be generated.
|
// If you remove this line, sources will not be generated.
|
||||||
withSourcesJar()
|
withSourcesJar()
|
||||||
|
|
||||||
sourceCompatibility = JavaVersion.VERSION_21
|
sourceCompatibility = JavaVersion.VERSION_21
|
||||||
targetCompatibility = JavaVersion.VERSION_21
|
targetCompatibility = JavaVersion.VERSION_21
|
||||||
}
|
}
|
||||||
|
|
||||||
jar {
|
jar {
|
||||||
inputs.property "archivesName", project.base.archivesName
|
inputs.property "archivesName", project.base.archivesName
|
||||||
|
manifest {
|
||||||
|
attributes(Map.of("Main-Class", "io.ib67.sfcraft.bundler.Main"))
|
||||||
|
}
|
||||||
|
from("LICENSE") {
|
||||||
|
rename { "${it}_${inputs.properties.archivesName}" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
from("LICENSE") {
|
remapJar {
|
||||||
rename { "${it}_${inputs.properties.archivesName}"}
|
input = tasks.shadowJar.archiveFile
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// configure the maven publication
|
// configure the maven publication
|
||||||
publishing {
|
publishing {
|
||||||
publications {
|
publications {
|
||||||
create("mavenJava", MavenPublication) {
|
create("mavenJava", MavenPublication) {
|
||||||
artifactId = project.archives_base_name
|
artifactId = project.archives_base_name
|
||||||
from components.java
|
from components.java
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// See https://docs.gradle.org/current/userguide/publishing_maven.html for information on how to set up publishing.
|
// See https://docs.gradle.org/current/userguide/publishing_maven.html for information on how to set up publishing.
|
||||||
repositories {
|
repositories {
|
||||||
// Add repositories to publish to here.
|
// Add repositories to publish to here.
|
||||||
// Notice: This block does NOT have the same function as the block in the top level.
|
// Notice: This block does NOT have the same function as the block in the top level.
|
||||||
// The repositories here will be used for publishing your artifact, not for
|
// The repositories here will be used for publishing your artifact, not for
|
||||||
// retrieving dependencies.
|
// retrieving dependencies.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+3
-3
@@ -1,5 +1,5 @@
|
|||||||
# Done to increase the memory available to gradle.
|
# Done to increase the memory available to gradle.
|
||||||
org.gradle.jvmargs=-Xmx1G
|
org.gradle.jvmargs=-Xmx4G
|
||||||
org.gradle.parallel=true
|
org.gradle.parallel=true
|
||||||
|
|
||||||
# Fabric Properties
|
# Fabric Properties
|
||||||
@@ -11,8 +11,8 @@ loom_version=1.11-SNAPSHOT
|
|||||||
|
|
||||||
# Mod Properties
|
# Mod Properties
|
||||||
mod_version=1.0.0
|
mod_version=1.0.0
|
||||||
maven_group=com.example
|
maven_group=io.ib67.sfcraft
|
||||||
archives_base_name=modid
|
archives_base_name=justbackup
|
||||||
|
|
||||||
# Dependencies
|
# Dependencies
|
||||||
fabric_version=0.129.0+1.21.8
|
fabric_version=0.129.0+1.21.8
|
||||||
@@ -3,6 +3,8 @@ package io.ib67.sfcraft;
|
|||||||
public record Backup(
|
public record Backup(
|
||||||
String name,
|
String name,
|
||||||
String backupKey,
|
String backupKey,
|
||||||
|
String type,
|
||||||
|
String from,
|
||||||
boolean incremental,
|
boolean incremental,
|
||||||
long sizeTotal
|
long sizeTotal
|
||||||
) {
|
) {
|
||||||
|
|||||||
@@ -0,0 +1,106 @@
|
|||||||
|
package io.ib67.sfcraft;
|
||||||
|
|
||||||
|
import com.mojang.brigadier.Command;
|
||||||
|
import com.mojang.brigadier.context.CommandContext;
|
||||||
|
import net.minecraft.server.command.ServerCommandSource;
|
||||||
|
import net.minecraft.text.ClickEvent;
|
||||||
|
import net.minecraft.text.Text;
|
||||||
|
|
||||||
|
import java.awt.*;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
|
||||||
|
public class BackupCommands {
|
||||||
|
protected final JustBackupMod mod;
|
||||||
|
protected Runnable restoreIssued;
|
||||||
|
|
||||||
|
public BackupCommands(JustBackupMod mod) {
|
||||||
|
this.mod = mod;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int cmdList(CommandContext<ServerCommandSource> ctx) {
|
||||||
|
var s = ctx.getSource();
|
||||||
|
s.sendFeedback(() -> Text.of("Tracked backups:"), false);
|
||||||
|
for (Backup value : mod.tracker.getTrackedBackups().values()) {
|
||||||
|
var restoreClick = Text.literal("[RESTORE]").withColor(Color.GREEN.getRGB())
|
||||||
|
.styled(it -> it.withClickEvent(new ClickEvent.SuggestCommand("/backup restore " + value.backupKey())));
|
||||||
|
var deleteClick = Text.literal("[DELETE]").withColor(Color.RED.getRGB())
|
||||||
|
.styled(it -> it.withClickEvent(new ClickEvent.SuggestCommand("/backup delete " + value.backupKey())));
|
||||||
|
var message = Text.literal(" - " + value.name() + " (" + value.backupKey() + ") " +
|
||||||
|
(value.incremental() ? "incremental" : "full") + " ").append(restoreClick).append(" ").append(deleteClick);
|
||||||
|
s.sendMessage(message);
|
||||||
|
s.sendMessage(Text.of(" Size: " + Math.floorDiv(value.sizeTotal(), 1024 * 1024) + "MiB"));
|
||||||
|
}
|
||||||
|
return Command.SINGLE_SUCCESS;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int cmdDelete(CommandContext<ServerCommandSource> ctx) {
|
||||||
|
var _backup = ctx.getArgument("backupName", String.class);
|
||||||
|
var backup = mod.tracker.getTrackedBackups().get(_backup);
|
||||||
|
mod.tracker.deleteBackup(backup);
|
||||||
|
ctx.getSource().sendMessage(Text.of("Backup " + _backup + " is deleted successfully."));
|
||||||
|
return Command.SINGLE_SUCCESS;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int cmdIssueBackup(CommandContext<ServerCommandSource> ctx) {
|
||||||
|
var s = ctx.getSource();
|
||||||
|
if (JustBackupMod.PERFORMING_BACKUP.get()) {
|
||||||
|
s.sendMessage(Text.of("You can't issue a backup because another backup is in progress."));
|
||||||
|
return Command.SINGLE_SUCCESS;
|
||||||
|
}
|
||||||
|
mod.issueBackup()
|
||||||
|
.whenComplete((backup, error) -> {
|
||||||
|
if (error != null) {
|
||||||
|
s.sendMessage(Text.of("Cannot backup, error: " + error.getMessage()));
|
||||||
|
} else {
|
||||||
|
s.sendMessage(Text.of("Backup created successfully! name: " + backup.name()));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return Command.SINGLE_SUCCESS;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int cmdSuspend(CommandContext<ServerCommandSource> ctx) {
|
||||||
|
mod.suspend = !mod.suspend;
|
||||||
|
ctx.getSource().sendMessage(Text.of("Backup disabled: " + mod.suspend));
|
||||||
|
return Command.SINGLE_SUCCESS;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int cmdRestore(CommandContext<ServerCommandSource> ctx) {
|
||||||
|
var s = ctx.getSource();
|
||||||
|
var pm = s.getServer().getPlayerManager();
|
||||||
|
var _backup = ctx.getArgument("backupName", String.class);
|
||||||
|
var backup = mod.tracker.getTrackedBackups().get(_backup);
|
||||||
|
if (backup == null) {
|
||||||
|
s.sendMessage(Text.of("Invalid backup " + _backup));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (restoreIssued != null) {
|
||||||
|
s.sendMessage(Text.of("Another restoration is in progress. try /backup cancel to cancel it"));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
s.sendMessage(Text.literal("Please restart your server to take changes"));
|
||||||
|
restoreIssued = () -> {
|
||||||
|
System.out.println("Recovering backup... ");
|
||||||
|
try{
|
||||||
|
var toOverride = Path.of(backup.from());
|
||||||
|
var old = Path.of(backup.from()+"_old");
|
||||||
|
Files.deleteIfExists(old);
|
||||||
|
Files.move(toOverride, old);
|
||||||
|
System.out.println("Moved your old save to "+old);
|
||||||
|
mod.tracker.recoverBackup(backup, toOverride);
|
||||||
|
System.out.println("Recover successfully!");
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
return Command.SINGLE_SUCCESS;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int cmdHelp(CommandContext<ServerCommandSource> serverCommandSourceCommandContext) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int cmdCancelRestore(CommandContext<ServerCommandSource> serverCommandSourceCommandContext) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
package io.ib67.sfcraft;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
public class BackupManager {
|
|
||||||
public List<Backup> getAllBackup() {
|
|
||||||
}
|
|
||||||
|
|
||||||
public void deleteBackup(Backup backup) {
|
|
||||||
}
|
|
||||||
|
|
||||||
public Backup performBackup() {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,27 +1,34 @@
|
|||||||
package io.ib67.sfcraft;
|
package io.ib67.sfcraft;
|
||||||
|
|
||||||
|
import io.ib67.sfcraft.bundler.Bundle;
|
||||||
|
import io.ib67.sfcraft.config.JustBackupConfig;
|
||||||
import io.ib67.sfcraft.mixin.LevelStorageAccessor;
|
import io.ib67.sfcraft.mixin.LevelStorageAccessor;
|
||||||
import io.ib67.sfcraft.strategy.BackupStrategy;
|
import io.ib67.sfcraft.strategy.BackupStrategy;
|
||||||
import lombok.SneakyThrows;
|
import lombok.SneakyThrows;
|
||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
import net.minecraft.server.MinecraftServer;
|
import net.minecraft.server.MinecraftServer;
|
||||||
|
|
||||||
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.format.DateTimeFormatter;
|
||||||
|
import java.util.concurrent.Callable;
|
||||||
|
import java.util.function.Supplier;
|
||||||
|
|
||||||
import static io.ib67.sfcraft.JustBackupMod.PERFORMING_BACKUP;
|
import static io.ib67.sfcraft.JustBackupMod.PERFORMING_BACKUP;
|
||||||
|
|
||||||
@Log4j2
|
@Log4j2
|
||||||
public record BackupWorker(
|
public record BackupWorker(
|
||||||
MinecraftServer server,
|
MinecraftServer server,
|
||||||
BackupManager manager
|
BackupStrategy strategy,
|
||||||
) implements Runnable {
|
JustBackupConfig config
|
||||||
|
) implements Supplier<Backup> {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public void run() {
|
public Backup get() {
|
||||||
if (server == null) {
|
if (server == null) {
|
||||||
log.warn("MinecraftServer is not initialized yet, not doing backup");
|
throw new IllegalStateException("MinecraftServer is not initialized yet, not doing backup");
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
int i = 0;
|
int i = 0;
|
||||||
while (!PERFORMING_BACKUP.compareAndSet(false, true)) {
|
while (!PERFORMING_BACKUP.compareAndSet(false, true)) {
|
||||||
@@ -36,8 +43,20 @@ public record BackupWorker(
|
|||||||
}
|
}
|
||||||
var session = ((LevelStorageAccessor) server).getSession();
|
var session = ((LevelStorageAccessor) server).getSession();
|
||||||
var path = session.getDirectory().getRootPath();
|
var path = session.getDirectory().getRootPath();
|
||||||
manager.performBackup();
|
var backupName = "Backup_" + DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(LocalDateTime.now());
|
||||||
strategy.createBackup(new WorldDir(Path.of(path)));
|
var backupParentRoot = Path.of(config.temporaryBackupDir());
|
||||||
|
if (Files.notExists(backupParentRoot)) Files.createDirectories(backupParentRoot);
|
||||||
|
var backupFile = backupParentRoot.resolve(backupName + ".jpack");
|
||||||
|
log.info("Creating bundle {}", backupName);
|
||||||
|
//todo diff
|
||||||
|
Path worldDir = Path.of(path);
|
||||||
|
new Bundle(new WorldDir(worldDir))
|
||||||
|
.compress(config.compress())
|
||||||
|
.allowGunzip(config.allowGunzip())
|
||||||
|
.buildBundle(backupFile);
|
||||||
|
var backup = strategy.createBackup(worldDir, backupFile);
|
||||||
|
Files.deleteIfExists(backupFile);
|
||||||
|
return backup;
|
||||||
} finally {
|
} finally {
|
||||||
PERFORMING_BACKUP.set(false);
|
PERFORMING_BACKUP.set(false);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,36 +2,58 @@ package io.ib67.sfcraft;
|
|||||||
|
|
||||||
import com.google.gson.Gson;
|
import com.google.gson.Gson;
|
||||||
import com.google.gson.GsonBuilder;
|
import com.google.gson.GsonBuilder;
|
||||||
|
import com.google.gson.reflect.TypeToken;
|
||||||
|
import com.mojang.brigadier.CommandDispatcher;
|
||||||
|
import com.mojang.brigadier.arguments.StringArgumentType;
|
||||||
import io.ib67.sfcraft.config.JustBackupConfig;
|
import io.ib67.sfcraft.config.JustBackupConfig;
|
||||||
import io.ib67.sfcraft.config.StorageOption;
|
import io.ib67.sfcraft.config.StorageOption;
|
||||||
import io.ib67.sfcraft.config.serializer.StorageOptionSerializer;
|
import io.ib67.sfcraft.config.serializer.StorageOptionSerializer;
|
||||||
import io.ib67.sfcraft.mixin.LevelStorageAccessor;
|
import io.ib67.sfcraft.mixin.LevelStorageAccessor;
|
||||||
import io.ib67.sfcraft.mixin.MixinMinecraftServer;
|
import io.ib67.sfcraft.mixin.MixinMinecraftServer;
|
||||||
import io.ib67.sfcraft.strategy.BackupStrategy;
|
import io.ib67.sfcraft.strategy.BackupStrategy;
|
||||||
|
import io.ib67.sfcraft.strategy.BackupTracker;
|
||||||
|
import io.ib67.sfcraft.strategy.LocalBackupStrategy;
|
||||||
|
import io.ib67.sfcraft.strategy.S3BackupStrategy;
|
||||||
import lombok.SneakyThrows;
|
import lombok.SneakyThrows;
|
||||||
import net.fabricmc.api.ModInitializer;
|
import net.fabricmc.api.ModInitializer;
|
||||||
|
|
||||||
|
import net.fabricmc.fabric.api.command.v2.CommandRegistrationCallback;
|
||||||
import net.fabricmc.fabric.api.event.lifecycle.v1.ServerLifecycleEvents;
|
import net.fabricmc.fabric.api.event.lifecycle.v1.ServerLifecycleEvents;
|
||||||
|
import net.minecraft.command.CommandRegistryAccess;
|
||||||
import net.minecraft.server.MinecraftServer;
|
import net.minecraft.server.MinecraftServer;
|
||||||
|
import net.minecraft.server.command.CommandManager;
|
||||||
|
import net.minecraft.server.command.ServerCommandSource;
|
||||||
import net.minecraft.server.world.ServerWorld;
|
import net.minecraft.server.world.ServerWorld;
|
||||||
|
import net.minecraft.text.Text;
|
||||||
|
import org.jetbrains.annotations.UnknownNullability;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import java.io.FileInputStream;
|
import java.io.FileInputStream;
|
||||||
|
import java.io.IOException;
|
||||||
import java.io.RandomAccessFile;
|
import java.io.RandomAccessFile;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
import java.time.format.DateTimeFormatter;
|
import java.time.format.DateTimeFormatter;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
import java.util.concurrent.CompletableFuture;
|
||||||
import java.util.concurrent.Executors;
|
import java.util.concurrent.Executors;
|
||||||
import java.util.concurrent.ScheduledExecutorService;
|
import java.util.concurrent.ScheduledExecutorService;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
|
||||||
|
import static net.minecraft.server.command.CommandManager.argument;
|
||||||
|
import static net.minecraft.server.command.CommandManager.literal;
|
||||||
|
|
||||||
public class JustBackupMod implements ModInitializer {
|
public class JustBackupMod implements ModInitializer {
|
||||||
public static final String MOD_ID = "justbackup";
|
public static final String MOD_ID = "justbackup";
|
||||||
public static final Logger LOGGER = LoggerFactory.getLogger(MOD_ID);
|
public static final Logger LOGGER = LoggerFactory.getLogger(MOD_ID);
|
||||||
|
/**
|
||||||
|
* This state indicates:
|
||||||
|
* 1. A minecraft auto-save is in-progress.
|
||||||
|
* 2. The backup is in-progress
|
||||||
|
*/
|
||||||
public static final AtomicBoolean PERFORMING_BACKUP = new AtomicBoolean(false);
|
public static final AtomicBoolean PERFORMING_BACKUP = new AtomicBoolean(false);
|
||||||
public static final Gson GSON = new GsonBuilder().setPrettyPrinting()
|
public static final Gson GSON = new GsonBuilder().setPrettyPrinting()
|
||||||
.registerTypeAdapter(StorageOption.class, new StorageOptionSerializer(
|
.registerTypeAdapter(StorageOption.class, new StorageOptionSerializer(
|
||||||
@@ -41,38 +63,95 @@ public class JustBackupMod implements ModInitializer {
|
|||||||
.create();
|
.create();
|
||||||
protected ScheduledExecutorService scheduledBackupExecutor;
|
protected ScheduledExecutorService scheduledBackupExecutor;
|
||||||
protected JustBackupConfig config;
|
protected JustBackupConfig config;
|
||||||
protected BackupStrategy strategy;
|
protected BackupTracker tracker;
|
||||||
|
protected Path backupIndexPath;
|
||||||
protected volatile MinecraftServer server;
|
protected volatile MinecraftServer server;
|
||||||
|
protected volatile boolean suspend;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public void onInitialize() {
|
public void onInitialize() {
|
||||||
LOGGER.info("Loading backup configuration");
|
LOGGER.info("Loading backup configuration");
|
||||||
|
backupIndexPath = Path.of("config").resolve("just_backup_index.json");
|
||||||
var configPath = Path.of("config").resolve("just_backup.json");
|
var configPath = Path.of("config").resolve("just_backup.json");
|
||||||
if (Files.notExists(configPath.getParent())) {
|
if (Files.notExists(configPath.getParent()))
|
||||||
Files.createDirectories(configPath.getParent());
|
Files.createDirectories(configPath.getParent());
|
||||||
}
|
if (Files.notExists(configPath))
|
||||||
if (Files.notExists(configPath)) {
|
|
||||||
saveConfig(configPath);
|
saveConfig(configPath);
|
||||||
}
|
if (Files.notExists(backupIndexPath))
|
||||||
|
Files.createFile(backupIndexPath);
|
||||||
|
|
||||||
config = GSON.fromJson(Files.readString(configPath), JustBackupConfig.class);
|
config = GSON.fromJson(Files.readString(configPath), JustBackupConfig.class);
|
||||||
this.strategy = createStrategy();
|
var strategy = createStrategy();
|
||||||
|
this.tracker = new BackupTracker(strategy, GSON.fromJson(Files.readString(backupIndexPath), new TypeToken<>() {
|
||||||
|
}));
|
||||||
LOGGER.info("Configuration has been loaded. Using storage option: {}", config.option().type());
|
LOGGER.info("Configuration has been loaded. Using storage option: {}", config.option().type());
|
||||||
scheduledBackupExecutor = Executors.newSingleThreadScheduledExecutor();
|
scheduledBackupExecutor = Executors.newSingleThreadScheduledExecutor();
|
||||||
|
registerHandlers();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void registerHandlers() {
|
||||||
ServerLifecycleEvents.SERVER_STARTED.register(server -> {
|
ServerLifecycleEvents.SERVER_STARTED.register(server -> {
|
||||||
LOGGER.info("Launching background threads...");
|
LOGGER.info("Launching background threads...");
|
||||||
this.server = server;
|
this.server = server;
|
||||||
scheduledBackupExecutor.scheduleAtFixedRate(new BackupWorker(server, strategy),
|
scheduledBackupExecutor.scheduleAtFixedRate(this::issueBackup,
|
||||||
config.backupIntervalMinutes(), config.backupIntervalMinutes(), TimeUnit.MINUTES);
|
config.backupIntervalMinutes(), config.backupIntervalMinutes(), TimeUnit.MINUTES);
|
||||||
});
|
});
|
||||||
|
ServerLifecycleEvents.SERVER_STOPPED.register(server -> {
|
||||||
|
try {
|
||||||
|
Files.deleteIfExists(Path.of(config.temporaryBackupDir()));
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
CommandRegistrationCallback.EVENT.register(this::registerCommand);
|
||||||
|
}
|
||||||
|
|
||||||
|
public CompletableFuture<Backup> issueBackup() {
|
||||||
|
if (suspend) {
|
||||||
|
return CompletableFuture.failedFuture(new IllegalStateException("Backup is temporarily disabled"));
|
||||||
|
}
|
||||||
|
server.getPlayerManager().broadcast(Text.literal("The server will begin the backup shortly, you may experience some lag."), false);
|
||||||
|
var worker = new BackupWorker(server, tracker, config);
|
||||||
|
return CompletableFuture.supplyAsync(worker, scheduledBackupExecutor).whenComplete(this::onBackupComplete);
|
||||||
|
}
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
private void onBackupComplete(@UnknownNullability Backup backup, @UnknownNullability Throwable throwable) {
|
||||||
|
if (throwable != null) {
|
||||||
|
server.getPlayerManager().broadcast(Text.literal("Backup failed. For administrators, please check your server console."), false);
|
||||||
|
throwable.printStackTrace();
|
||||||
|
} else {
|
||||||
|
Files.writeString(backupIndexPath, GSON.toJson(tracker.getTrackedBackups()));
|
||||||
|
server.getPlayerManager().broadcast(Text.literal("Backup success. " + backup), false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void registerCommand(
|
||||||
|
CommandDispatcher<ServerCommandSource> dispatcher,
|
||||||
|
CommandRegistryAccess registry, CommandManager.RegistrationEnvironment env) {
|
||||||
|
var commandHandler = new BackupCommands(this);
|
||||||
|
dispatcher.register(literal("backup")
|
||||||
|
.requires(it -> it.hasPermissionLevel(3))
|
||||||
|
.then(literal("help").executes(commandHandler::cmdHelp))
|
||||||
|
.then(literal("list").executes(commandHandler::cmdList))
|
||||||
|
.then(literal("cancelrestore").executes(commandHandler::cmdCancelRestore))
|
||||||
|
.then(literal("delete")
|
||||||
|
.then(argument("backupName", StringArgumentType.greedyString())
|
||||||
|
.executes(commandHandler::cmdDelete)))
|
||||||
|
.then(literal("restore")
|
||||||
|
.then(argument("backupName", StringArgumentType.greedyString())
|
||||||
|
.executes(commandHandler::cmdRestore)))
|
||||||
|
.then(literal("create").executes(commandHandler::cmdIssueBackup))
|
||||||
|
.then(literal("suspend").executes(commandHandler::cmdSuspend))
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private BackupStrategy createStrategy() {
|
private BackupStrategy createStrategy() {
|
||||||
switch (config.option()) {
|
return switch (config.option()) {
|
||||||
case StorageOption.Local local -> {
|
case StorageOption.Local local -> new LocalBackupStrategy(local);
|
||||||
|
case StorageOption.S3 s3 -> new S3BackupStrategy(s3);
|
||||||
}
|
};
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -83,8 +162,14 @@ public class JustBackupMod implements ModInitializer {
|
|||||||
60,
|
60,
|
||||||
5,
|
5,
|
||||||
true,
|
true,
|
||||||
new StorageOption.Local("backups")
|
true,
|
||||||
|
"./.backup_cache",
|
||||||
|
new StorageOption.Local("backups", 0)
|
||||||
);
|
);
|
||||||
|
var path = Path.of(cfg.temporaryBackupDir());
|
||||||
|
if (Files.notExists(path)) {
|
||||||
|
Files.createDirectories(path);
|
||||||
|
}
|
||||||
Files.writeString(configPath, GSON.toJson(cfg));
|
Files.writeString(configPath, GSON.toJson(cfg));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -42,7 +42,9 @@ public record WorldDir(
|
|||||||
public List<Path> otherFiles() {
|
public List<Path> otherFiles() {
|
||||||
var otherDimensions = otherDimensions();
|
var otherDimensions = otherDimensions();
|
||||||
try (var f = Files.walk(root)) {
|
try (var f = Files.walk(root)) {
|
||||||
return f.filter(it -> otherDimensions.stream().noneMatch(a -> it.startsWith(a.root)))
|
return f.filter(it -> Files.isRegularFile(it)
|
||||||
|
&& otherDimensions.stream().noneMatch(a -> it.startsWith(a.root)))
|
||||||
|
.filter(it -> !it.getFileName().toString().contains("session.lock"))
|
||||||
.toList();
|
.toList();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -50,7 +52,9 @@ public record WorldDir(
|
|||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public List<Path> everything() {
|
public List<Path> everything() {
|
||||||
try (var f = Files.walk(root)) {
|
try (var f = Files.walk(root)) {
|
||||||
return f.toList();
|
return f.filter(Files::isRegularFile)
|
||||||
|
.filter(it -> !it.getFileName().toString().contains("session.lock"))
|
||||||
|
.toList();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
package io.ib67.sfcraft.bundler;
|
||||||
|
|
||||||
|
import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
|
||||||
|
import io.ib67.sfcraft.WorldDir;
|
||||||
|
import io.netty.buffer.Unpooled;
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.SneakyThrows;
|
||||||
|
import lombok.experimental.Accessors;
|
||||||
|
import org.apache.commons.lang3.Validate;
|
||||||
|
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.nio.channels.FileChannel;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.StandardOpenOption;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Accessors(fluent = true)
|
||||||
|
@Data
|
||||||
|
public class Bundle {
|
||||||
|
public static final int MAGIC = 0xBACEBACE;
|
||||||
|
public static final int COMPRESSED_MAGIC = 0xBACEAAAA;
|
||||||
|
protected final WorldDir dir;
|
||||||
|
protected boolean compress = true;
|
||||||
|
protected boolean allowGunzip = true;
|
||||||
|
protected boolean allowParallel = false;
|
||||||
|
|
||||||
|
|
||||||
|
public Bundle(WorldDir dir) {
|
||||||
|
this.dir = dir;
|
||||||
|
}
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
public static void unbundleFile(Path bundle, Path out) {
|
||||||
|
try (var bundlerReader = new SerialBundlerReader(bundle)) {
|
||||||
|
while (bundlerReader.hasEntry()) {
|
||||||
|
var entry = bundlerReader.readEntry();
|
||||||
|
var target = out.resolve(entry.fileName());
|
||||||
|
Files.createDirectories(target.getParent());
|
||||||
|
bundlerReader.writeTo(FileChannel.open(target, StandardOpenOption.WRITE,
|
||||||
|
StandardOpenOption.CREATE_NEW), entry);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
public void buildBundle(Path backupFile) {
|
||||||
|
checkCondition();
|
||||||
|
var sortedEntries = dir.everything().stream().sorted((a, b) -> {
|
||||||
|
boolean aMatch = a.endsWith(".mca") || a.endsWith(".dat");
|
||||||
|
boolean bMatch = b.endsWith(".mca") || b.endsWith(".dat");
|
||||||
|
|
||||||
|
if (aMatch && !bMatch) return -1;
|
||||||
|
if (!aMatch && bMatch) return 1;
|
||||||
|
return a.compareTo(b);
|
||||||
|
}).map(dir.root()::relativize).toList();
|
||||||
|
if (Files.notExists(backupFile.getParent()))
|
||||||
|
Files.createDirectories(backupFile.getParent());
|
||||||
|
SerialBundlerWriter writer;
|
||||||
|
if (compress) {
|
||||||
|
var fout = Files.newOutputStream(backupFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
|
||||||
|
writeMagic(fout, null, true);
|
||||||
|
var out = new ZstdOutputStreamNoFinalizer(fout);
|
||||||
|
writer = new SerialBundlerWriter(out);
|
||||||
|
} else {
|
||||||
|
var chan = FileChannel.open(backupFile);
|
||||||
|
writer = new SerialBundlerWriter(chan, 0);
|
||||||
|
writeMagic(null, chan, false);
|
||||||
|
}
|
||||||
|
// todo parallelism
|
||||||
|
writeEntries(sortedEntries, writer, allowGunzip && compress);
|
||||||
|
writer.close();
|
||||||
|
|
||||||
|
try (var backup = FileChannel.open(backupFile, StandardOpenOption.WRITE, StandardOpenOption.APPEND);
|
||||||
|
var bos = new ByteArrayOutputStream();
|
||||||
|
var metadata = new EntryOutputStream(new ZstdOutputStreamNoFinalizer(bos))) {
|
||||||
|
var entries = writer.getEntries();
|
||||||
|
for (BundlerEntry entry : entries) {
|
||||||
|
metadata.write(entry);
|
||||||
|
}
|
||||||
|
metadata.flush();
|
||||||
|
var result = bos.toByteArray();
|
||||||
|
var buffer = ByteBuffer.allocate(result.length +4 + 4);
|
||||||
|
buffer.put(result);
|
||||||
|
buffer.putInt(entries.size());
|
||||||
|
buffer.putInt(result.length);
|
||||||
|
writeFully(backup, buffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static void writeMagic(OutputStream out, FileChannel channel, boolean compress) throws IOException {
|
||||||
|
var magic = compress ? Bundle.COMPRESSED_MAGIC : Bundle.MAGIC;
|
||||||
|
if (channel != null) {
|
||||||
|
channel.write(ByteBuffer.allocate(4).putInt(magic).flip());
|
||||||
|
} else {
|
||||||
|
out.write(ByteBuffer.allocate(4).putInt(magic).array());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void writeFully(FileChannel chan, ByteBuffer buf) throws IOException {
|
||||||
|
buf.flip();
|
||||||
|
while (buf.hasRemaining()) {
|
||||||
|
chan.write(buf);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void checkCondition() {
|
||||||
|
if (allowParallel) {
|
||||||
|
Validate.isTrue(!compress && !allowGunzip);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void writeEntries(
|
||||||
|
List<Path> sortedEntries,
|
||||||
|
SerialBundlerWriter writer,
|
||||||
|
boolean allowGunzip) throws IOException {
|
||||||
|
for (Path sortedEntry : sortedEntries) {
|
||||||
|
var fileName = sortedEntry.getFileName();
|
||||||
|
var shouldGunzip = allowGunzip && (fileName.endsWith("mca") || fileName.endsWith("dat"));
|
||||||
|
if (shouldGunzip) {
|
||||||
|
writer.writeGunzip(dir.root().resolve(sortedEntry), sortedEntry.toString());
|
||||||
|
} else {
|
||||||
|
writer.writePlain(dir.root().resolve(sortedEntry), sortedEntry.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -6,5 +6,4 @@ public record BundlerEntry(
|
|||||||
long fileLength,
|
long fileLength,
|
||||||
long offset
|
long offset
|
||||||
) {
|
) {
|
||||||
public static final int MAGIC = 0xBACEBACE;
|
|
||||||
}
|
}
|
||||||
@@ -24,7 +24,7 @@ public class EntryInputStream extends FilterInputStream {
|
|||||||
var gunzipped = len < 0;
|
var gunzipped = len < 0;
|
||||||
len = Math.abs(len);
|
len = Math.abs(len);
|
||||||
var name = new String(readNBytes(len)).intern();
|
var name = new String(readNBytes(len)).intern();
|
||||||
var buf = ByteBuffer.wrap(readNBytes(8));
|
var buf = ByteBuffer.wrap(readNBytes(8 + 8));
|
||||||
var fileLen = buf.getLong();
|
var fileLen = buf.getLong();
|
||||||
var fileOffset = buf.getLong();
|
var fileOffset = buf.getLong();
|
||||||
return new BundlerEntry(name, gunzipped, fileLen, fileOffset);
|
return new BundlerEntry(name, gunzipped, fileLen, fileOffset);
|
||||||
|
|||||||
@@ -20,10 +20,6 @@ public class EntryOutputStream extends FilterOutputStream {
|
|||||||
super(out);
|
super(out);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void writeMagic() throws IOException {
|
|
||||||
out.write(BundlerEntry.MAGIC);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void write(BundlerEntry entry) throws IOException {
|
public void write(BundlerEntry entry) throws IOException {
|
||||||
var fileName = entry.fileName().getBytes(StandardCharsets.UTF_8);
|
var fileName = entry.fileName().getBytes(StandardCharsets.UTF_8);
|
||||||
var buf = ByteBuffer.allocate(4).putInt(entry.unGzipped() ? fileName.length * -1 : fileName.length);
|
var buf = ByteBuffer.allocate(4).putInt(entry.unGzipped() ? fileName.length * -1 : fileName.length);
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
package io.ib67.sfcraft.bundler;
|
||||||
|
|
||||||
|
import io.ib67.kiwi.ArgOpts;
|
||||||
|
import io.ib67.sfcraft.WorldDir;
|
||||||
|
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
|
||||||
|
public class Main {
|
||||||
|
public static void main(String[] args) {
|
||||||
|
var opts = ArgOpts.builder()
|
||||||
|
.args(args).description("Tools for .jpack format bundle").programName("jpack").build();
|
||||||
|
var in = opts.string("in", "Input. Can be directory or jpack bundle", null);
|
||||||
|
var out = opts.string("out", "Output. Can be jpack bundle or directory", null);
|
||||||
|
var help = opts.bool("help", false);
|
||||||
|
var allowGunzip = opts.bool("gunzip",
|
||||||
|
"Should we recompress NBT files for maximum compression rate (depends on compress)", true);
|
||||||
|
var compress = opts.bool("compress", "Should we compress?", true);
|
||||||
|
var parallel = opts.bool("parallel", "Use parallelized bundler. This conflicts with compress options", false);
|
||||||
|
if (help) {
|
||||||
|
opts.printHelp(System.out);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (allowGunzip && !compress) {
|
||||||
|
System.out.println("allowGunzip depends on compress feature.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (in == null || out == null) {
|
||||||
|
System.out.println("Both in and out should be provided");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
var inPath = Path.of(in);
|
||||||
|
var outPath = Path.of(out);
|
||||||
|
if (!Files.exists(inPath)) {
|
||||||
|
System.out.println("The input isn't exists.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (Files.exists(outPath)) {
|
||||||
|
System.out.println("The output is already exists. Please move it to other places");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (in.endsWith(".jpack")) {
|
||||||
|
Bundle.unbundleFile(inPath, outPath);
|
||||||
|
} else if (out.endsWith(".jpack")) {
|
||||||
|
if (!Files.isDirectory(inPath))
|
||||||
|
throw new IllegalArgumentException("Your input should be a world directory.");
|
||||||
|
new Bundle(new WorldDir(inPath))
|
||||||
|
.allowGunzip(allowGunzip)
|
||||||
|
.compress(compress)
|
||||||
|
.allowParallel(parallel)
|
||||||
|
.buildBundle(outPath);
|
||||||
|
} else {
|
||||||
|
System.out.println("Invalid arguments. You should provide a pair of jpack and directory to act as in/output.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -19,31 +19,37 @@ public class SerialBundlerReader implements Closeable {
|
|||||||
protected final EntryInputStream metadataIn;
|
protected final EntryInputStream metadataIn;
|
||||||
protected final byte[] buffer = new byte[4096];
|
protected final byte[] buffer = new byte[4096];
|
||||||
protected boolean entryRead;
|
protected boolean entryRead;
|
||||||
|
protected int entries;
|
||||||
|
|
||||||
public SerialBundlerReader(InputStream in, InputStream metadataIn) {
|
// public SerialBundlerReader(InputStream in, InputStream metadataIn) {
|
||||||
this.in = Objects.requireNonNull(in);
|
// this.in = Objects.requireNonNull(in);
|
||||||
this.metadataIn = new EntryInputStream(Objects.requireNonNull(metadataIn));
|
// this.metadataIn = new EntryInputStream(Objects.requireNonNull(metadataIn));
|
||||||
inChannel = null;
|
// inChannel = null;
|
||||||
}
|
// }
|
||||||
|
|
||||||
public SerialBundlerReader(Path path) throws IOException {
|
public SerialBundlerReader(Path path) throws IOException {
|
||||||
var raf = new RandomAccessFile(path.toFile(), "r");
|
var raf = new RandomAccessFile(path.toFile(), "r");
|
||||||
var totalLen = Files.size(path);
|
var totalLen = Files.size(path);
|
||||||
|
var magic = raf.readInt();
|
||||||
|
var compressed = magic == Bundle.COMPRESSED_MAGIC;
|
||||||
|
if (!compressed && magic != Bundle.MAGIC) throw new IOException("Invalid bundle file");
|
||||||
|
|
||||||
|
// read length
|
||||||
raf.seek(totalLen - 4);
|
raf.seek(totalLen - 4);
|
||||||
var metadataLength = raf.readInt();
|
var metadataLength = raf.readInt();
|
||||||
var metadataBegin = totalLen - 4 - metadataLength;
|
raf.seek(totalLen - 8);
|
||||||
|
entries = raf.readInt();
|
||||||
|
// read metadata
|
||||||
|
var metadataBegin = totalLen - 8 - metadataLength;
|
||||||
var metadataBytes = new byte[metadataLength];
|
var metadataBytes = new byte[metadataLength];
|
||||||
raf.seek(metadataBegin);
|
raf.seek(metadataBegin);
|
||||||
raf.readFully(metadataBytes);
|
raf.readFully(metadataBytes);
|
||||||
|
|
||||||
this.metadataIn = new EntryInputStream(new ZstdInputStreamNoFinalizer(new ByteArrayInputStream(metadataBytes)));
|
this.metadataIn = new EntryInputStream(new ZstdInputStreamNoFinalizer(new ByteArrayInputStream(metadataBytes)));
|
||||||
if (ByteBuffer.wrap(metadataIn.readNBytes(4)).getInt() != BundlerEntry.MAGIC) {
|
|
||||||
throw new IllegalArgumentException("Invalid Bundler entry");
|
|
||||||
}
|
|
||||||
// check compressed
|
// check compressed
|
||||||
raf.seek(0);
|
raf.seek(4);
|
||||||
var fis = Channels.newInputStream(raf.getChannel());
|
var fis = Channels.newInputStream(raf.getChannel());
|
||||||
if (raf.readInt() == Zstd.magicNumber()) {
|
if (compressed) {
|
||||||
in = new ZstdInputStreamNoFinalizer(fis);
|
in = new ZstdInputStreamNoFinalizer(fis);
|
||||||
inChannel = null;
|
inChannel = null;
|
||||||
} else {
|
} else {
|
||||||
@@ -54,12 +60,14 @@ public class SerialBundlerReader implements Closeable {
|
|||||||
|
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public boolean hasEntry() {
|
public boolean hasEntry() {
|
||||||
return metadataIn.available() > 0;
|
return entries > 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
public BundlerEntry readEntry() throws IOException {
|
public BundlerEntry readEntry() throws IOException {
|
||||||
|
if (!hasEntry()) throw new IOException("All entries are read");
|
||||||
var entry = metadataIn.readEntry();
|
var entry = metadataIn.readEntry();
|
||||||
entryRead = true;
|
entryRead = true;
|
||||||
|
entries--;
|
||||||
return entry;
|
return entry;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import java.nio.channels.FileChannel;
|
|||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.nio.file.StandardOpenOption;
|
import java.nio.file.StandardOpenOption;
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.zip.GZIPInputStream;
|
import java.util.zip.GZIPInputStream;
|
||||||
@@ -28,12 +29,14 @@ public class SerialBundlerWriter implements Closeable {
|
|||||||
public SerialBundlerWriter(OutputStream out) {
|
public SerialBundlerWriter(OutputStream out) {
|
||||||
Objects.requireNonNull(out);
|
Objects.requireNonNull(out);
|
||||||
this.out = out;
|
this.out = out;
|
||||||
|
this.entries = new ArrayList<>();
|
||||||
channel = null;
|
channel = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
public SerialBundlerWriter(FileChannel channel, long offset) {
|
public SerialBundlerWriter(FileChannel channel, long offset) {
|
||||||
Objects.requireNonNull(channel);
|
Objects.requireNonNull(channel);
|
||||||
this.offset = offset;
|
this.offset = offset;
|
||||||
|
this.entries = new ArrayList<>();
|
||||||
this.channel = channel;
|
this.channel = channel;
|
||||||
out = null;
|
out = null;
|
||||||
}
|
}
|
||||||
@@ -50,10 +53,12 @@ public class SerialBundlerWriter implements Closeable {
|
|||||||
public void writePlain(Path path, String fileName) throws IOException {
|
public void writePlain(Path path, String fileName) throws IOException {
|
||||||
if (channel != null) {
|
if (channel != null) {
|
||||||
writeZC(path, fileName);
|
writeZC(path, fileName);
|
||||||
return;
|
} else {
|
||||||
}
|
var current = offset;
|
||||||
try (var in = Files.newInputStream(path)) {
|
try (var in = Files.newInputStream(path)) {
|
||||||
in.transferTo(out);
|
in.transferTo(out);
|
||||||
|
}
|
||||||
|
entries.add(new BundlerEntry(fileName, false, Files.size(path), current));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,8 +7,9 @@ import java.util.List;
|
|||||||
public record JustBackupConfig(
|
public record JustBackupConfig(
|
||||||
long backupIntervalMinutes,
|
long backupIntervalMinutes,
|
||||||
int keepBackups,
|
int keepBackups,
|
||||||
boolean allowParallel,
|
boolean allowGunzip,
|
||||||
boolean compress,
|
boolean compress,
|
||||||
|
String temporaryBackupDir,
|
||||||
StorageOption option
|
StorageOption option
|
||||||
) {
|
) {
|
||||||
public JustBackupConfig {
|
public JustBackupConfig {
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
package io.ib67.sfcraft.config;
|
package io.ib67.sfcraft.config;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.Validate;
|
||||||
|
|
||||||
public sealed interface StorageOption {
|
public sealed interface StorageOption {
|
||||||
String type();
|
String type();
|
||||||
|
|
||||||
@@ -8,20 +10,32 @@ public sealed interface StorageOption {
|
|||||||
String accessKey,
|
String accessKey,
|
||||||
String secretKey,
|
String secretKey,
|
||||||
String endpoint,
|
String endpoint,
|
||||||
String bucket
|
String bucket,
|
||||||
|
String prefix,
|
||||||
|
boolean enforcePathStyle,
|
||||||
|
int retryAmount
|
||||||
) implements StorageOption {
|
) implements StorageOption {
|
||||||
@Override
|
@Override
|
||||||
public String type() {
|
public String type() {
|
||||||
return "s3";
|
return "s3";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public S3 {
|
||||||
|
Validate.isTrue(retryAmount >= 0);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
record Local(
|
record Local(
|
||||||
String saveDir
|
String saveDir,
|
||||||
|
long diskSizeQuotaBytes
|
||||||
) implements StorageOption {
|
) implements StorageOption {
|
||||||
@Override
|
@Override
|
||||||
public String type() {
|
public String type() {
|
||||||
return "local";
|
return "local";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Local {
|
||||||
|
Validate.isTrue(diskSizeQuotaBytes >= 0);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2,16 +2,20 @@ package io.ib67.sfcraft.strategy;
|
|||||||
|
|
||||||
import io.ib67.sfcraft.Backup;
|
import io.ib67.sfcraft.Backup;
|
||||||
import io.ib67.sfcraft.WorldDir;
|
import io.ib67.sfcraft.WorldDir;
|
||||||
|
import io.ib67.sfcraft.bundler.Bundle;
|
||||||
|
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
|
|
||||||
public interface BackupStrategy {
|
public interface BackupStrategy {
|
||||||
/**
|
/**
|
||||||
* create a backup
|
* create a backup
|
||||||
* @param dir world to be bundled
|
|
||||||
* @return a key to this backup
|
* @return a key to this backup
|
||||||
*/
|
*/
|
||||||
Backup createBackup(WorldDir dir, boolean compress);
|
Backup createBackup(Path source, Path bundle);
|
||||||
|
|
||||||
void recoverBackup(Backup backup, Path restorePath);
|
void recoverBackup(Backup backup, Path restorePath);
|
||||||
|
|
||||||
|
void deleteBackup(Backup backup);
|
||||||
|
|
||||||
|
boolean isAvailable();
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
package io.ib67.sfcraft.strategy;
|
||||||
|
|
||||||
|
import io.ib67.sfcraft.Backup;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
public class BackupTracker implements BackupStrategy {
|
||||||
|
protected final BackupStrategy underlying;
|
||||||
|
@Getter
|
||||||
|
protected final Map<String, Backup> trackedBackups;
|
||||||
|
|
||||||
|
public BackupTracker(BackupStrategy underlying, Map<String, Backup> trackedBackups) {
|
||||||
|
this.underlying = Objects.requireNonNull(underlying);
|
||||||
|
this.trackedBackups = trackedBackups == null ? new HashMap<>() : trackedBackups;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Backup createBackup(Path from, Path pathToBundle) {
|
||||||
|
var result = underlying.createBackup(from, pathToBundle);
|
||||||
|
trackedBackups.put(result.backupKey(), result);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void recoverBackup(Backup backup, Path restorePath) {
|
||||||
|
underlying.recoverBackup(backup, restorePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void deleteBackup(Backup backup) {
|
||||||
|
underlying.deleteBackup(backup);
|
||||||
|
trackedBackups.remove(backup.backupKey());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isAvailable() {
|
||||||
|
return underlying.isAvailable();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,14 +4,12 @@ import com.github.luben.zstd.Zstd;
|
|||||||
import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
|
import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
|
||||||
import io.ib67.sfcraft.Backup;
|
import io.ib67.sfcraft.Backup;
|
||||||
import io.ib67.sfcraft.WorldDir;
|
import io.ib67.sfcraft.WorldDir;
|
||||||
import io.ib67.sfcraft.bundler.BundlerEntry;
|
import io.ib67.sfcraft.bundler.*;
|
||||||
import io.ib67.sfcraft.bundler.EntryOutputStream;
|
|
||||||
import io.ib67.sfcraft.bundler.SerialBundlerReader;
|
|
||||||
import io.ib67.sfcraft.bundler.SerialBundlerWriter;
|
|
||||||
import io.ib67.sfcraft.config.StorageOption;
|
import io.ib67.sfcraft.config.StorageOption;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.SneakyThrows;
|
import lombok.SneakyThrows;
|
||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
|
import org.apache.commons.lang3.Validate;
|
||||||
|
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
@@ -25,80 +23,61 @@ import java.time.format.DateTimeFormatter;
|
|||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
|
||||||
@Log4j2
|
@Log4j2
|
||||||
@RequiredArgsConstructor
|
|
||||||
public class LocalBackupStrategy implements BackupStrategy {
|
public class LocalBackupStrategy implements BackupStrategy {
|
||||||
protected final StorageOption.Local option;
|
protected final StorageOption.Local option;
|
||||||
protected final Path backupParentRoot = Path.of(option.saveDir());
|
protected final Path backupParentRoot;
|
||||||
protected final boolean allowGunzip;
|
|
||||||
|
@SneakyThrows
|
||||||
|
public LocalBackupStrategy(StorageOption.Local option) {
|
||||||
|
this.option = option;
|
||||||
|
backupParentRoot = Path.of(option.saveDir());
|
||||||
|
Files.createDirectories(backupParentRoot);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public Backup createBackup(WorldDir dir, boolean compress) {
|
public Backup createBackup(Path from, Path pathToBundle) {
|
||||||
var sortedEntries = dir.everything().stream().sorted((a, b) -> {
|
var result = new Backup(
|
||||||
boolean aMatch = a.endsWith(".mca") || a.endsWith(".dat");
|
pathToBundle.getFileName().toString(),
|
||||||
boolean bMatch = b.endsWith(".mca") || b.endsWith(".dat");
|
pathToBundle.getFileName().toString(),
|
||||||
|
"local", from.toString(), false, Files.size(pathToBundle));
|
||||||
if (aMatch && !bMatch) return -1;
|
Files.move(pathToBundle, backupParentRoot.resolve(pathToBundle.getFileName()));
|
||||||
if (!aMatch && bMatch) return 1;
|
return result;
|
||||||
return a.compareTo(b);
|
|
||||||
}).map(dir.root()::relativize).toList();
|
|
||||||
var backupName = "Backup_" + DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(LocalDateTime.now());
|
|
||||||
var backupFile = backupParentRoot.resolve(backupName + ".jbp");
|
|
||||||
SerialBundlerWriter writer;
|
|
||||||
if (compress) {
|
|
||||||
var out = new ZstdOutputStreamNoFinalizer(Files.newOutputStream(backupFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE));
|
|
||||||
writer = new SerialBundlerWriter(out);
|
|
||||||
} else {
|
|
||||||
var chan = FileChannel.open(backupFile);
|
|
||||||
writer = new SerialBundlerWriter(chan, 0);
|
|
||||||
}
|
|
||||||
writeEntries(sortedEntries, writer, allowGunzip && compress);
|
|
||||||
writer.close();
|
|
||||||
|
|
||||||
try (var backup = FileChannel.open(backupFile);
|
|
||||||
var bos = new ByteArrayOutputStream();
|
|
||||||
var metadata = new EntryOutputStream(new ZstdOutputStreamNoFinalizer(bos))) {
|
|
||||||
metadata.writeMagic();
|
|
||||||
for (BundlerEntry entry : writer.getEntries()) {
|
|
||||||
metadata.write(entry);
|
|
||||||
}
|
|
||||||
var result = bos.toByteArray();
|
|
||||||
var buffer = ByteBuffer.allocate(result.length + 4);
|
|
||||||
buffer.put(result);
|
|
||||||
buffer.putLong(result.length);
|
|
||||||
backup.write(buffer);
|
|
||||||
}
|
|
||||||
return new Backup(backupName, backupFile.getFileName().toString(), false, Files.size(backupFile));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void writeEntries(
|
|
||||||
List<Path> sortedEntries,
|
|
||||||
SerialBundlerWriter writer,
|
|
||||||
boolean allowGunzip) throws IOException {
|
|
||||||
for (Path sortedEntry : sortedEntries) {
|
|
||||||
var fileName = sortedEntry.getFileName();
|
|
||||||
var shouldGunzip = allowGunzip && (fileName.endsWith("mca") || fileName.endsWith("dat"));
|
|
||||||
if (shouldGunzip) {
|
|
||||||
writer.writeGunzip(sortedEntry, sortedEntry.toString());
|
|
||||||
} else {
|
|
||||||
writer.writePlain(sortedEntry, sortedEntry.toString());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public void recoverBackup(Backup backup, Path restorePath) {
|
public void recoverBackup(Backup backup, Path restorePath) {
|
||||||
// check if it is compressed
|
// check if it is compressed
|
||||||
var backupFile = backupParentRoot.resolve(backup.backupKey());
|
var backupFile = backupParentRoot.resolve(backup.backupKey());
|
||||||
if (Files.notExists(backupFile)) throw new IllegalArgumentException("Backup " + backupFile + " does not exist");
|
if (Files.notExists(backupFile)) throw new IllegalArgumentException("Backup " + backupFile + " does not exist");
|
||||||
try (var bundlerReader = new SerialBundlerReader(backupFile)) {
|
Bundle.unbundleFile(backupFile, restorePath);
|
||||||
while (bundlerReader.hasEntry()) {
|
}
|
||||||
var entry = bundlerReader.readEntry();
|
|
||||||
var target = restorePath.resolve(entry.fileName());
|
@SneakyThrows
|
||||||
bundlerReader.writeTo(FileChannel.open(target), entry);
|
@Override
|
||||||
|
public void deleteBackup(Backup backup) {
|
||||||
|
Validate.isTrue("local".equals(backup.backupKey()));
|
||||||
|
var bundlePath = backupParentRoot.resolve(backup.backupKey());
|
||||||
|
Files.deleteIfExists(bundlePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
@SneakyThrows
|
||||||
|
public boolean isAvailable() {
|
||||||
|
if (option.diskSizeQuotaBytes() == 0) return true;
|
||||||
|
try (var f = Files.walk(backupParentRoot)) {
|
||||||
|
if (f.mapToLong(it->{
|
||||||
|
try {
|
||||||
|
return Files.size(it);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}).sum() >= option.diskSizeQuotaBytes()) {
|
||||||
|
log.warn("Have no space to set another backup! ");
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,104 @@
|
|||||||
|
package io.ib67.sfcraft.strategy;
|
||||||
|
|
||||||
|
import io.ib67.sfcraft.Backup;
|
||||||
|
import io.ib67.sfcraft.WorldDir;
|
||||||
|
import io.ib67.sfcraft.bundler.Bundle;
|
||||||
|
import io.ib67.sfcraft.config.StorageOption;
|
||||||
|
import io.minio.BucketExistsArgs;
|
||||||
|
import io.minio.MinioClient;
|
||||||
|
import io.minio.RemoveObjectArgs;
|
||||||
|
import io.minio.UploadObjectArgs;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.SneakyThrows;
|
||||||
|
import lombok.extern.log4j.Log4j2;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
|
||||||
|
@Log4j2
|
||||||
|
public class S3BackupStrategy implements BackupStrategy {
|
||||||
|
protected final StorageOption.S3 option;
|
||||||
|
protected final MinioClient minio;
|
||||||
|
protected final AtomicBoolean uploading = new AtomicBoolean(false);
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
public S3BackupStrategy(StorageOption.S3 option) {
|
||||||
|
this.option = option;
|
||||||
|
this.minio = MinioClient.builder()
|
||||||
|
.endpoint(URI.create(option.endpoint()).toURL())
|
||||||
|
.credentials(option.accessKey(), option.secretKey())
|
||||||
|
.region(option.region())
|
||||||
|
.build();
|
||||||
|
if (option.enforcePathStyle()) {
|
||||||
|
minio.disableVirtualStyleEndpoint();
|
||||||
|
} else {
|
||||||
|
minio.enableVirtualStyleEndpoint();
|
||||||
|
}
|
||||||
|
log.info("Testing S3 credentials");
|
||||||
|
var exists = minio.bucketExists(BucketExistsArgs.builder().bucket(option.bucket()).build());
|
||||||
|
if (!exists) throw new IllegalStateException("Bucket '" + option.bucket() + "' does not exist");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
@SneakyThrows
|
||||||
|
public Backup createBackup(Path from, Path pathToBundle) {
|
||||||
|
if (!uploading.compareAndSet(false, true)) {
|
||||||
|
throw new IllegalStateException("Last upload is not done yet! keep waiting...");
|
||||||
|
}
|
||||||
|
int i = 0;
|
||||||
|
do {
|
||||||
|
try {
|
||||||
|
var object = option.prefix() + "/" + pathToBundle.getFileName().toString();
|
||||||
|
minio.uploadObject(UploadObjectArgs.builder()
|
||||||
|
.contentType("application/octet-stream")
|
||||||
|
.object(object)
|
||||||
|
.filename(pathToBundle.toString())
|
||||||
|
.build());
|
||||||
|
//todo also calculate sha?
|
||||||
|
return new Backup(pathToBundle.getFileName().toString(),
|
||||||
|
object, "s3",
|
||||||
|
from.toString(),
|
||||||
|
false,
|
||||||
|
Files.size(pathToBundle));
|
||||||
|
} catch (IOException e) {
|
||||||
|
if (i >= option.retryAmount()) {
|
||||||
|
throw new IllegalStateException("Cannot upload " + pathToBundle + " to S3");
|
||||||
|
}
|
||||||
|
var nextTry = Math.pow(2, i++);
|
||||||
|
log.error("Error uploading S3. trying in {}s later. ({}/{})", nextTry, i, option.retryAmount(), e);
|
||||||
|
Thread.sleep((long) (nextTry * 1000));
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new IllegalStateException("Unrecoverable exception occurred when uploading " + pathToBundle, e);
|
||||||
|
} finally {
|
||||||
|
uploading.set(false);
|
||||||
|
}
|
||||||
|
} while (true);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void recoverBackup(Backup backup, Path restorePath) {
|
||||||
|
//todo http range
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
@SneakyThrows
|
||||||
|
public void deleteBackup(Backup backup) {
|
||||||
|
minio.removeObject(RemoveObjectArgs.builder()
|
||||||
|
.bucket(backup.backupKey()).build());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isAvailable() {
|
||||||
|
if (uploading.get()) {
|
||||||
|
log.warn("Last S3 Backup is still uploading!");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user