icybear/justbackup
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

overhaul thread model and backup reliability

Browse Source 
Thread model:
  - replace the IOState CAS state machine with a StampedLock: region IO
    (RegionFileStorage.write/flush, now both guarded via @WrapMethod) holds
    the read lock, backups hold the write lock; the old witness-restore
    pattern could wedge the state permanently
  - run saveEverything on the main thread (was called from the backup
    executor), with shutdown-aware waiting; skip autoSave/saveEverything
    while a backup holds the write lock so the main thread never blocks
  - broadcast player messages from the main thread, honor `broadcast`

  Backup correctness:
  - watcher reported the parent directory instead of the changed file, so
    every incremental backup was empty; change sets are now deduplicated,
    atomically drained and merged back on failure; OVERFLOW forces a full
    backup; watch roots are re-registered when their inode changes (world
    format conversion orphaned the watches)
  - fix the region reassembly pipeline: zlib was "decompressed" with
    DeflaterInputStream, writeChunk dropped the 5-byte chunk header,
    the length field was off by one, coordinates were transposed, and
    sector padding was wrong on 4096-byte multiples — every reassembled
    chunk in old bundles is unrecoverable garbage
  - full backups now collect the file list after saving, off-thread

  New features:
  - reflink (FICLONE) staging via FFM: on CoW filesystems the write lock
    is only held while cloning, packing/uploading runs unlocked
  - chained restore: restoring an incremental automatically applies its
    full base and every incremental in between, in order
  - generation-based rotation implementing keepBackups; whole generations
    are deleted so an incremental never loses its base
  - Backup records carry subject/createdAt/incremental for chaining

  Reliability:
  - atomic index persistence (tmp + ATOMIC_MOVE), ConcurrentHashMap for
    the tracked-backup index, fsync bundles before storing
  - BundleReader: overlay extraction for chains, zip-slip guard, fix
    reversed createSymbolicLink args, EOF instead of infinite loop on
    truncated bundles
  - LocalBackupStrategy.deleteBackup compared the wrong field and could
    never delete; S3 retries now catch SdkException (retry was dead code),
    drop ifNoneMatch for idempotent retries
  - ordered shutdown: SERVER_STOPPING awaits in-flight backups before the
    final world save; SERVER_STOPPED runs restore, closes storage and
    cleans the temp dir with isolated error handling
  - fix /backup suspend being registered under /backup create

  Tests: region round-trip byte-comparison, BundleReader overlay/zip-slip/
  symlink/truncation, backup chain resolution and rotation.

  Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
iceBear67 committed 2026-07-17 19:35:43 +08:00
1 parent 592c79ed90
commit 34f7d15566
25 files changed
+1418 -367
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/main/java/io/ib67/sfcraft/JustBackupMod.java
+155 -62
View File
@@ -4,6 +4,7 @@ import com.github.luben.zstd.Zstd;
import com.google.gson.reflect.TypeToken;
import io.ib67.sfcraft.config.JustBackupConfig;
import io.ib67.sfcraft.config.StorageOption;
import io.ib67.sfcraft.strategy.BackupChains;
import io.ib67.sfcraft.strategy.BackupStrategy;
import io.ib67.sfcraft.strategy.BackupTracker;
import io.ib67.sfcraft.strategy.LocalBackupStrategy;
@@ -25,14 +26,16 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class JustBackupMod implements ModInitializer {
public class JustBackupMod implements ModInitializer, FileWatcherThread.Listener {
public static final String MOD_ID = "justbackup";
public static final Logger LOGGER = LoggerFactory.getLogger(MOD_ID);
protected final Map<String, BackupSubject> backupSubjects = new HashMap<>();
@@ -44,6 +47,8 @@ public class JustBackupMod implements ModInitializer {
protected JustBackupConfig config;
protected BackupTracker tracker;
protected Path backupIndexPath;
protected Path tmpDir;
protected List<Path> excludedRoots;
protected FileWatcherThread watcherThread;
@Override
@@ -62,33 +67,45 @@ public class JustBackupMod implements ModInitializer {
@SneakyThrows
private void doInitialize() {
LOGGER.info("Loading backup configuration");
Path config1 = Path.of("config");
backupIndexPath = config1.resolve("just_backup_index.json");
var configPath = config1.resolve("just_backup.json");
Path configDir = Path.of("config");
backupIndexPath = configDir.resolve("just_backup_index.json");
var configPath = configDir.resolve("just_backup.json");
if (Files.notExists(configPath.getParent()))
Files.createDirectories(configPath.getParent());
if (Files.notExists(configPath))
saveConfig(configPath);
if (Files.notExists(backupIndexPath))
Files.createFile(backupIndexPath);
config = Globals.GSON.fromJson(Files.readString(configPath), JustBackupConfig.class);
config.backupSubjects().forEach((k, v) -> {
var subjectPath = Path.of(v);
if (Files.notExists(subjectPath)) {
LOGGER.error("Backup _subject {} does not exist.", v);
LOGGER.error("Backup subject {} does not exist.", v);
return;
}
backupSubjects.put(k, new BackupSubject(k, subjectPath, new ArrayList<>()));
backupSubjects.put(k, new BackupSubject(k, subjectPath));
});
if (config.zstdDictPath() != null && !config.zstdDictPath().isEmpty()) {
zstdDict = Files.readAllBytes(Path.of(config.zstdDictPath()));
}
tmpDir = Path.of(config.temporaryBackupDir());
excludedRoots = new ArrayList<>();
excludedRoots.add(tmpDir);
if (config.option() instanceof StorageOption.Local local) {
excludedRoots.add(Path.of(local.saveDir()));
}
var strategy = createStrategy();
this.tracker = new BackupTracker(strategy, Globals.GSON.fromJson(Files.readString(backupIndexPath), new TypeToken<>() {
}));
Map<String, Backup> initialIndex = Files.exists(backupIndexPath)
? Globals.GSON.fromJson(Files.readString(backupIndexPath), new TypeToken<Map<String, Backup>>() {
})
: null;
this.tracker = new BackupTracker(strategy, initialIndex, backupIndexPath);
LOGGER.info("Configuration has been loaded. Using storage option: {}", config.option().type());
scheduledBackupExecutor = Executors.newSingleThreadScheduledExecutor();
scheduledBackupExecutor = Executors.newSingleThreadScheduledExecutor(r -> {
var t = new Thread(r, "JustBackup/Worker");
t.setDaemon(false);
return t;
});
registerWatchService();
registerHandlers();
}
@@ -98,7 +115,7 @@ public class JustBackupMod implements ModInitializer {
config.backupSubjects().entrySet().stream()
.map(it -> Map.entry(it.getKey(), Path.of(it.getValue())))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)),
this::handleFileChanges
this
);
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
if (watcherThread.getState() != Thread.State.TERMINATED) watcherThread.interrupt();
@@ -106,12 +123,16 @@ public class JustBackupMod implements ModInitializer {
watcherThread.start();
}
private void handleFileChanges(String _subject, Path path) {
synchronized (backupSubjects) {
var subject = backupSubjects.get(_subject);
if (subject != null)
subject.changedFiles().add(path); // check null
}
@Override
public void onChange(String subjectName, Path file) {
var subject = backupSubjects.get(subjectName);
if (subject != null) subject.addChanged(file);
}
@Override
public void onOverflow(String subjectName) {
var subject = backupSubjects.get(subjectName);
if (subject != null) subject.forceFullNext();
}
@SneakyThrows
@@ -130,96 +151,168 @@ public class JustBackupMod implements ModInitializer {
}
lastBackupServerTicks = server.getTickCount();
if (!suspend) backupAll(config.incremental()).thenAccept(it -> {
LOGGER.info("Backup success!");
it.forEach((k, v) -> LOGGER.info(k + ": " + v));
LOGGER.info("Scheduled backup finished.");
it.forEach((k, v) -> LOGGER.info("{}: {}", k, v));
});
},
config.backupIntervalMinutes(), config.backupIntervalMinutes(), TimeUnit.MINUTES);
});
ServerLifecycleEvents.SERVER_STOPPED.register(server -> {
try {
Files.deleteIfExists(Path.of(config.temporaryBackupDir()));
} catch (IOException e) {
throw new RuntimeException(e);
}
});
var command = new BackupCommands(this);
CommandRegistrationCallback.EVENT.register(command::registerCommand);
ServerLifecycleEvents.SERVER_STOPPED.register(server -> {
LOGGER.info("Shutting down the executor");
// Runs at the head of stopServer, before the final world save: in-flight backups must
// release the world IO lock before that save, or the flush would park forever.
ServerLifecycleEvents.SERVER_STOPPING.register(server -> {
suspend = true;
scheduledBackupExecutor.shutdown();
if (command.restoreIssued != null) {
command.restoreIssued.run();
try {
if (!scheduledBackupExecutor.awaitTermination(15, TimeUnit.SECONDS)) {
LOGGER.info("Waiting up to 10 minutes for the in-flight backup to finish...");
if (!scheduledBackupExecutor.awaitTermination(10, TimeUnit.MINUTES)) {
LOGGER.error("In-flight backup did not finish in time; its bundle may be incomplete.");
}
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
tracker.close();
});
ServerLifecycleEvents.SERVER_STOPPED.register(server -> {
if (command.restoreIssued != null) {
try {
command.restoreIssued.run();
} catch (Exception e) {
LOGGER.error("Backup restoration failed", e);
}
}
try {
tracker.close();
} catch (Exception e) {
LOGGER.error("Error while closing the backup storage", e);
}
cleanTemporaryDir();
if (watcherThread != null && watcherThread.getState() != Thread.State.TERMINATED) {
watcherThread.interrupt();
}
});
}
/**
* Removes only artifacts we created (staging clones, leftover bundles, temp downloads).
* Never deletes the directory itself: a misconfigured temporaryBackupDir pointing at the
* server root must not wipe the server.
*/
private void cleanTemporaryDir() {
try {
FsUtil.deleteRecursively(tmpDir.resolve("staging"));
} catch (IOException e) {
LOGGER.warn("Cannot clean staging dir", e);
}
if (Files.notExists(tmpDir)) return;
try (Stream<Path> files = Files.list(tmpDir)) {
files.filter(Files::isRegularFile)
.filter(p -> {
var n = p.getFileName().toString();
return (n.startsWith("backup_") && n.endsWith(".swb.zst")) || n.startsWith("s3_");
})
.forEach(p -> {
try {
Files.deleteIfExists(p);
} catch (IOException e) {
LOGGER.warn("Cannot delete leftover {}", p, e);
}
});
} catch (IOException e) {
LOGGER.warn("Cannot clean temporary backup dir", e);
}
}
/**
* Logs the message and, when enabled and a server is up, broadcasts it to players from
* the main thread (PlayerList is not safe to touch from the backup executor).
*/
public void broadcast(Component message) {
LOGGER.info(message.getString());
if (!config.broadcast()) return;
var s = server;
if (s == null || !s.isRunning()) return;
s.submit(() -> s.getPlayerList().broadcastSystemMessage(message, false));
}
public CompletableFuture<Map<String, Backup>> backupAll(boolean incremental) {
if (backupSubjects.isEmpty()) return CompletableFuture.completedFuture(Map.of());
var map = new HashMap<String, CompletableFuture<Backup>>();
server.getPlayerList().broadcastSystemMessage(
Component.literal(" BACKUP >> ").withColor(Color.RED.getRGB()).withStyle(it -> it.withBold(true))
.append(Component.nullToEmpty("The server is performing a backup, you may experience some lag.")),
false
);
broadcast(Component.literal(" BACKUP >> ").withColor(Color.RED.getRGB()).withStyle(it -> it.withBold(true))
.append(Component.nullToEmpty("The server is performing a backup, you may experience some lag.")));
for (var entry : backupSubjects.entrySet()) {
map.put(entry.getKey(), issueBackup(entry.getKey(), incremental));
}
return CompletableFuture.allOf(map.values().toArray(new CompletableFuture[0]))
.thenApply(it -> {
var _map = new HashMap<String, Backup>();
var results = new HashMap<String, Backup>();
map.forEach((k, v) -> {
var result = v.join();
LOGGER.info("Backup information of {}: {}", k, result);
_map.put(k, result);
results.put(k, result);
});
return _map;
return results;
});
}
public CompletableFuture<Backup> issueBackup(String subjectName, boolean incremental) {
var subject = backupSubjects.get(subjectName);
if (subject == null)
return CompletableFuture.failedFuture(new IllegalArgumentException("Unknown backup _subject: " + subjectName));
if (!incremental) {
try (var s = Files.walk(subject.root())) {
subject = new BackupSubject(subject.name(), subject.root(), s.filter(Files::isRegularFile)
.filter(it -> !it.getFileName().toString().equals("session.lock"))
.toList());
} catch (IOException e) {
return CompletableFuture.failedFuture(new IllegalArgumentException("Cannot perform full backup for " + subjectName, e));
}
}
var finalSubject = subject;
return CompletableFuture.failedFuture(new IllegalArgumentException("Unknown backup subject: " + subjectName));
if (config.allowGunzip()) LOGGER.info("Chunk reassembler is enabled!");
var worker = new BackupWorker(server, tracker, Path.of(config.temporaryBackupDir()),
subject, c ->
var worker = new BackupWorker(server, tracker, tmpDir, subject, incremental, excludedRoots, c ->
c.allowGunzip(config.allowGunzip())
.relativeRoot(finalSubject.root())
.allocator(ByteBufAllocator.DEFAULT)
.maxWorkers(config.compressionWorkerThreads())
.compressionLevel(config.compressionLevel())
.dictionary(zstdDict)
);
return CompletableFuture.supplyAsync(worker, scheduledBackupExecutor).whenComplete(this::onBackupComplete);
return CompletableFuture.supplyAsync(() -> {
if (!tracker.isAvailable()) {
throw new IllegalStateException("Backup storage is not available (quota exceeded?), skipping backup of " + subjectName);
}
return worker.get();
}, scheduledBackupExecutor).whenComplete(this::onBackupComplete);
}
@SneakyThrows
private void onBackupComplete(@UnknownNullability Backup backup, @UnknownNullability Throwable throwable) {
if (throwable != null) {
server.getPlayerList().broadcastSystemMessage(Component.literal("Backup failed. For administrators, please check your server console."), false);
LOGGER.error(throwable.getMessage(), throwable);
broadcast(Component.literal("Backup failed. For administrators, please check your server console."));
LOGGER.error("Backup failed", throwable);
} else {
Files.writeString(backupIndexPath, Globals.GSON.toJson(tracker.getTrackedBackups()));
server.getPlayerList().broadcastSystemMessage(Component.literal("Backup success. " + backup), false);
broadcast(Component.literal("Backup success. ").append(backup.toText()));
rotate(backup.subject());
}
}
/**
* Deletes whole generations (a full backup plus its incrementals) beyond the newest
* {@code keepBackups} ones. Entries without chain metadata are never rotated away.
*/
private void rotate(String subjectName) {
if (subjectName == null) return;
try {
var victims = BackupChains.rotationVictims(
tracker.getTrackedBackups().values(), subjectName, config.keepBackups());
for (var victim : victims) {
LOGGER.info("Rotating out old backup {} ({})", victim.backupKey(), victim.incremental() ? "INCR" : "FULL");
try {
tracker.deleteBackup(victim);
} catch (Exception e) {
LOGGER.warn("Cannot delete old backup {}", victim.backupKey(), e);
}
}
} catch (Exception e) {
LOGGER.warn("Backup rotation failed for {}", subjectName, e);
}
}
private BackupStrategy createStrategy() {
return switch (config.option()) {
case StorageOption.Local local -> new LocalBackupStrategy(local);
case StorageOption.S3 s3 -> new S3BackupStrategy(s3, Path.of(config.temporaryBackupDir()));
case StorageOption.S3 s3 -> new S3BackupStrategy(s3, tmpDir);
};
}
@@ -245,4 +338,4 @@ public class JustBackupMod implements ModInitializer {
}
Files.writeString(configPath, Globals.GSON.toJson(cfg));
}
}
}