overhaul thread model and backup reliability

Thread model:
  - replace the IOState CAS state machine with a StampedLock: region IO
    (RegionFileStorage.write/flush, now both guarded via @WrapMethod) holds
    the read lock, backups hold the write lock; the old witness-restore
    pattern could wedge the state permanently
  - run saveEverything on the main thread (was called from the backup
    executor), with shutdown-aware waiting; skip autoSave/saveEverything
    while a backup holds the write lock so the main thread never blocks
  - broadcast player messages from the main thread, honor `broadcast`

  Backup correctness:
  - watcher reported the parent directory instead of the changed file, so
    every incremental backup was empty; change sets are now deduplicated,
    atomically drained and merged back on failure; OVERFLOW forces a full
    backup; watch roots are re-registered when their inode changes (world
    format conversion orphaned the watches)
  - fix the region reassembly pipeline: zlib was "decompressed" with
    DeflaterInputStream, writeChunk dropped the 5-byte chunk header,
    the length field was off by one, coordinates were transposed, and
    sector padding was wrong on 4096-byte multiples — every reassembled
    chunk in old bundles is unrecoverable garbage
  - full backups now collect the file list after saving, off-thread

  New features:
  - reflink (FICLONE) staging via FFM: on CoW filesystems the write lock
    is only held while cloning, packing/uploading runs unlocked
  - chained restore: restoring an incremental automatically applies its
    full base and every incremental in between, in order
  - generation-based rotation implementing keepBackups; whole generations
    are deleted so an incremental never loses its base
  - Backup records carry subject/createdAt/incremental for chaining

  Reliability:
  - atomic index persistence (tmp + ATOMIC_MOVE), ConcurrentHashMap for
    the tracked-backup index, fsync bundles before storing
  - BundleReader: overlay extraction for chains, zip-slip guard, fix
    reversed createSymbolicLink args, EOF instead of infinite loop on
    truncated bundles
  - LocalBackupStrategy.deleteBackup compared the wrong field and could
    never delete; S3 retries now catch SdkException (retry was dead code),
    drop ifNoneMatch for idempotent retries
  - ordered shutdown: SERVER_STOPPING awaits in-flight backups before the
    final world save; SERVER_STOPPED runs restore, closes storage and
    cleans the temp dir with isolated error handling
  - fix /backup suspend being registered under /backup create

  Tests: region round-trip byte-comparison, BundleReader overlay/zip-slip/
  symlink/truncation, backup chain resolution and rotation.

  Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
iceBear67 committed 2026-07-17 19:35:43 +08:00
1 parent 592c79ed90
commit 34f7d15566
25 files changed
+1418 -367

No files matched your search

@@ -6,7 +6,9 @@ import io.ib67.sfcraft.bundler.region.RegionParser;
import io.netty.buffer.ByteBufAllocator;
import lombok.Builder;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -32,8 +34,14 @@ public class BundleReader {
this.dict = dict;
}
/**
* Extracts a bundle into {@code destination}. Existing files are overwritten, which is
* what makes chained restores work: apply the full base first, then each incremental in
* order — later bundles win.
*/
public void extract(Path bundle, Path destination) throws IOException {
var readMaps = new HashMap<String, Long>();
var destRoot = destination.toAbsolutePath().normalize();
try (var zstd = new ZstdInputStreamNoFinalizer(Files.newInputStream(bundle, StandardOpenOption.READ));
var in = new EntryInputStream(zstd)) {
if (dict != null) zstd.setDict(dict);
@@ -42,62 +50,101 @@ public class BundleReader {
var time = entry.time();
if (time < minTime || time > maxTime) {
System.err.println("Skipping entry " + entry.name() + " (too old or too new)");
skipExactly(in, entry.length());
continue;
}
var lastSeen = readMaps.get(entry.name());
if (lastSeen != null && skipOldEntry && entry.time() < lastSeen) {
System.err.println("Skipping entry " + entry.name() + " (older than )");
System.err.println("Skipping entry " + entry.name() + " (older than seen)");
skipExactly(in, entry.length());
continue;
}
readMaps.put(entry.name(), time);
var path = destination.resolve(entry.name()); // todo prevent path underflow
var path = resolveSafely(destRoot, entry.name());
var parentDir = path.getParent();
if (Files.notExists(parentDir)) Files.createDirectories(parentDir);
if ((entry.attribute() & BundleEntry.ATTR_SYMLINK) != 0) {
var toRead = (int) entry.length();
var strArr = new byte[toRead];
while ((toRead -= in.read(strArr, (int) (entry.length() - toRead), toRead)) > 0) ;
var target = new String(strArr);
var targetPath = Path.of(target);
Files.createSymbolicLink(targetPath, path);
var strArr = new byte[(int) entry.length()];
readExactly(in, strArr);
var targetPath = Path.of(new String(strArr));
Files.deleteIfExists(path);
Files.createSymbolicLink(path, targetPath);
} else if ((entry.attribute() & BundleEntry.ATTR_REASSEMBLE) != 0) {
var entryData = allocator.buffer();
var toRead = entry.length();
var buffer = this.buffer;
while (toRead > 0) {
var read = in.read(buffer, 0, (int) Math.min(buffer.length, toRead));
toRead -= read;
entryData.writeBytes(buffer, 0, read);
}
try (var reassembler = new RegionParser(entryData, allocator)) {
var result = reassembler.writeReassembled(RegionFile.CompressType.ZLIB);
try (var fc = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
result.readBytes(fc, 0, result.readableBytes());
} finally {
result.release();
try {
var toRead = entry.length();
var buffer = this.buffer;
while (toRead > 0) {
var read = in.read(buffer, 0, (int) Math.min(buffer.length, toRead));
if (read < 0) throw new EOFException("Truncated bundle while reading " + entry.name());
toRead -= read;
entryData.writeBytes(buffer, 0, read);
}
} catch (Exception e) {
try (var ch = FileChannel.open(Path.of("dump.mca"), StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
entryData.readerIndex(0);
entryData.readBytes(ch, 0, entryData.readableBytes());
try (var reassembler = new RegionParser(entryData, allocator)) {
var result = reassembler.writeReassembled(RegionFile.CompressType.ZLIB);
try (var fc = FileChannel.open(path, StandardOpenOption.CREATE,
StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) {
result.readBytes(fc, 0, result.readableBytes());
} finally {
result.release();
}
} catch (Exception e) {
var dump = Files.createTempFile("justbackup_dump_", ".mca");
try (var ch = FileChannel.open(dump, StandardOpenOption.WRITE)) {
entryData.readerIndex(0);
entryData.readBytes(ch, 0, entryData.readableBytes());
}
throw new IOException("Error while reassembling " + entry.name() + ", dumped to " + dump, e);
}
throw new IOException("Error while reassembling " + entry.name(), e);
} finally {
entryData.release();
}
} else {
writePlain(path, entry, in);
}
}
}
}
/**
* Resolves an entry name inside the destination, refusing names that escape it
* (absolute paths or {@code ..} traversal).
*/
private static Path resolveSafely(Path destRoot, String entryName) throws IOException {
var resolved = destRoot.resolve(entryName).normalize();
if (!resolved.startsWith(destRoot)) {
throw new IOException("Entry " + entryName + " escapes the destination directory, refusing to extract");
}
return resolved;
}
private static void readExactly(InputStream in, byte[] target) throws IOException {
int off = 0;
while (off < target.length) {
int read = in.read(target, off, target.length - off);
if (read < 0) throw new EOFException("Truncated bundle");
off += read;
}
}
private void skipExactly(InputStream in, long length) throws IOException {
var buffer = this.buffer;
while (length > 0) {
int read = in.read(buffer, 0, (int) Math.min(buffer.length, length));
if (read < 0) throw new EOFException("Truncated bundle");
length -= read;
}
}
private void writePlain(Path path, BundleEntry entry, EntryInputStream in) throws IOException {
var buffer = this.buffer;
try (var fc = Files.newOutputStream(path, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
try (var fc = Files.newOutputStream(path, StandardOpenOption.CREATE,
StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) {
var toRead = entry.length();
while (toRead > 0) {
var read = in.read(buffer, 0, (int) Math.min(buffer.length, toRead));
if (read < 0) throw new EOFException("Truncated bundle while reading " + entry.name());
toRead -= read;
fc.write(buffer, 0, read);
}
@@ -12,9 +12,12 @@ import lombok.Builder;
import lombok.SneakyThrows;
import java.io.*;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.*;
import java.util.concurrent.*;
import java.util.function.UnaryOperator;
@@ -131,14 +134,40 @@ public class BundleWriter implements Closeable {
Iterable<Path> path,
UnaryOperator<BundleWriterBuilder> config
) throws IOException {
try (var writer = config.apply(BundleWriter.builder())
.outputStream(Files.newOutputStream(pathToBundle))
.build()) {
for (var p : path) {
if (!Files.isRegularFile(p) || Files.size(p) == 0) continue;
var s = p.toString();
writer.write(p, (s.contains("region") && s.endsWith(".mca")) || s.endsWith(".dat"));
try (var channel = FileChannel.open(pathToBundle,
StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) {
try (var writer = config.apply(BundleWriter.builder())
.outputStream(new NonClosingOutputStream(Channels.newOutputStream(channel)))
.build()) {
for (var p : path) {
if (!Files.isRegularFile(p) || Files.size(p) == 0) continue;
var s = p.toString();
writer.write(p, (s.contains("region") && s.endsWith(".mca")) || s.endsWith(".dat"));
}
}
// the zstd epilogue is flushed by writer.close(); fsync before closing so a crash
// right after "backup success" cannot leave a hollow bundle behind
channel.force(true);
}
}
/**
* Lets the zstd stream be closed (writing its epilogue) without closing the underlying
* channel, so the caller can fsync it afterwards.
*/
private static final class NonClosingOutputStream extends FilterOutputStream {
NonClosingOutputStream(OutputStream out) {
super(out);
}
@Override
public void write(byte[] b, int off, int len) throws IOException {
out.write(b, off, len);
}
@Override
public void close() throws IOException {
out.flush();
}
}
@@ -191,8 +220,10 @@ public class BundleWriter implements Closeable {
result.readBytes(out, length);
result.release();
// all done
} catch (IOException e) {
System.err.println("Failed to parse region " + path + ": " + e);
} catch (Exception e) {
// any parse failure (not only IOException) must fall back to a plain copy instead
// of failing the whole backup
System.err.println("Failed to parse region " + path + ", storing it as-is: " + e);
finalResult.release();
writePlain(path);
return;
@@ -205,7 +236,8 @@ public class BundleWriter implements Closeable {
private void writePlain(Path path) throws IOException {
var rPath = relativeRoot.toAbsolutePath().relativize(path.toAbsolutePath());
var outputStream = this.outputStream;
outputStream.writeEntryHeader(rPath.toString(), (short) 0, Files.size(path), 0);
outputStream.writeEntryHeader(rPath.toString(), (short) 0, Files.size(path),
Files.getLastModifiedTime(path).toMillis());
var read = 0;
try (var fs = Files.newInputStream(path)) {
var buffer = BundleWriter.buffer.get();
@@ -45,31 +45,31 @@ public class RegionFile implements Closeable {
return (short) ((z & 0b111111) << 6 | (x & 0b111111));
}
// this retains the input bytebuf.
public void writeChunk(int chunkX, int chunkZ, CompressType type, ByteBuf data) throws IOException {
if (chunkX > 32 || chunkZ > 32 || chunkX < 0 || chunkZ < 0)
if (chunkX >= 32 || chunkZ >= 32 || chunkX < 0 || chunkZ < 0)
throw new IllegalArgumentException("Invalid chunkX+chunkZ: " + chunkX + " " + chunkZ);
var key = getChunkKey(chunkX, chunkZ);
var buf = allocator.buffer(5 + data.readableBytes());
ByteBuf payload;
if (type == CompressType.NONE) {
buf.writeInt(data.readableBytes());
buf.writeByte(type.index);
buf.writeBytes(data);
payload = allocator.buffer(data.readableBytes());
payload.writeBytes(data);
} else {
try (var out = findCompressor(type, new ByteBufOutputStream(buf))) {
while (data.readableBytes() > 0) data.readBytes(out, data.readableBytes());
var head = allocator.buffer(5);
head.writeInt(buf.readableBytes());
head.writeByte(type.index);
var composite = allocator.compositeBuffer();
composite.addComponent(true, head);
composite.addComponent(true, buf);
payload = allocator.buffer();
try (var out = findCompressor(type, new ByteBufOutputStream(payload))) {
data.readBytes(out, data.readableBytes());
} catch (IOException e) {
buf.release();
payload.release();
throw e;
}
}
regionData.put(key, buf);
var buf = allocator.buffer(5 + payload.readableBytes());
// vanilla region format: the length field also counts the compression type byte
buf.writeInt(payload.readableBytes() + 1);
buf.writeByte(type.index);
buf.writeBytes(payload);
payload.release();
var previous = regionData.put(key, buf);
if (previous != null) previous.release();
}
private OutputStream findCompressor(CompressType type, OutputStream out) throws IOException {
@@ -82,7 +82,7 @@ public class RegionFile implements Closeable {
}
public void updateChunkTime(int chunkX, int chunkZ, int timestamp) {
if (chunkX > 32 || chunkZ > 32 || chunkX < 0 || chunkZ < 0)
if (chunkX >= 32 || chunkZ >= 32 || chunkX < 0 || chunkZ < 0)
throw new IllegalArgumentException("Invalid chunkX+chunkZ: " + chunkX + " " + chunkZ);
var index = (chunkX + 32 * chunkZ) * 4;
this.timestamp.setInt(index, timestamp);
@@ -105,7 +105,7 @@ public class RegionFile implements Closeable {
var chunk = entry.getValue();
var size = chunk.readableBytes();
var sizeSectors = Math.ceilDiv(size, 4096);
var zeros = 4096 - (size % 4096);
var zeros = (4096 - (size % 4096)) % 4096;
for (int i = 0; i < zeros; i++) {
chunk.writeByte(0);
}
@@ -10,8 +10,8 @@ import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.zip.DeflaterInputStream;
import java.util.zip.GZIPInputStream;
import java.util.zip.InflaterInputStream;
public class RegionParser implements Closeable {
protected final ByteBuf source;
@@ -28,7 +28,9 @@ public class RegionParser implements Closeable {
try (var fc = FileChannel.open(file, StandardOpenOption.READ)) {
var toRead = size;
while (toRead > 0) {
toRead -= source.writeBytes(fc, 0L, (int) size);
var read = source.writeBytes(fc, (long) (size - toRead), toRead);
if (read < 0) throw new EOFException("Region file " + file + " shrank while reading");
toRead -= read;
}
}
}
@@ -98,8 +100,9 @@ public class RegionParser implements Closeable {
var key = chunks.getLong(i);
var chunkXZ = readChunkXZ(key);
var entry = readEntry(key);
short x = (short) (chunkXZ & 0x3F);
short z = (short) ((chunkXZ >>> 6) & 0x3F);
// packed as (x << 6) | z in readAllValidChunks
short x = (short) ((chunkXZ >>> 6) & 0x3F);
short z = (short) (chunkXZ & 0x3F);
var compressType = readChunk(entry, compressed);
try (var decompressor = getDecompressorByType(compressType, compressed)) {
while (uncompressed.writeBytes(decompressor, 4096) != -1) ;
@@ -116,7 +119,7 @@ public class RegionParser implements Closeable {
private InputStream getDecompressorByType(int compressionType, ByteBuf in) throws IOException {
return switch (compressionType) {
case 1 -> new GZIPInputStream(new ByteBufInputStream(in));
case 2 -> new DeflaterInputStream(new ByteBufInputStream(in));
case 2 -> new InflaterInputStream(new ByteBufInputStream(in));
case 3 -> new ByteBufInputStream(in);
case 4 -> new LZ4BlockInputStream(new ByteBufInputStream(in));
// user defined compression algorithm, not going to support.
@@ -0,0 +1,88 @@
package io.ib67.bundler;
import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
import io.ib67.sfcraft.bundler.BundleReader;
import io.ib67.sfcraft.bundler.BundleWriter;
import io.ib67.sfcraft.bundler.EntryOutputStream;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.EOFException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
public class TestBundleReader {
@Test
public void testChainedOverlayRestore(@TempDir Path tmp) throws IOException {
var src = Files.createDirectories(tmp.resolve("src"));
Files.writeString(src.resolve("a.txt"), "version-1-that-is-long");
Files.writeString(src.resolve("b.txt"), "untouched");
var fullBundle = tmp.resolve("full.swb.zst");
BundleWriter.createBundle(fullBundle, List.of(src.resolve("a.txt"), src.resolve("b.txt")),
c -> c.relativeRoot(src));
// shorter content verifies the overlay truncates instead of leaving a stale tail
Files.writeString(src.resolve("a.txt"), "v2");
var incrBundle = tmp.resolve("incr.swb.zst");
BundleWriter.createBundle(incrBundle, List.of(src.resolve("a.txt")),
c -> c.relativeRoot(src));
var dest = tmp.resolve("dest");
var reader = BundleReader.builder().build();
reader.extract(fullBundle, dest);
reader.extract(incrBundle, dest);
assertEquals("v2", Files.readString(dest.resolve("a.txt")));
assertEquals("untouched", Files.readString(dest.resolve("b.txt")));
}
@Test
public void testZipSlipIsRejected(@TempDir Path tmp) throws IOException {
var bundle = tmp.resolve("evil.swb.zst");
var payload = "owned".getBytes();
try (var zstd = new ZstdOutputStreamNoFinalizer(Files.newOutputStream(bundle), 3);
var out = new EntryOutputStream(zstd)) {
out.writeEntryHeader("../evil.txt", (short) 0, payload.length, System.currentTimeMillis());
out.write(payload);
}
var dest = tmp.resolve("dest");
var ex = assertThrows(IOException.class,
() -> BundleReader.builder().build().extract(bundle, dest));
assertTrue(ex.getMessage().contains("escapes"), ex.getMessage());
assertFalse(Files.exists(tmp.resolve("evil.txt")));
}
@Test
public void testSymlinkIsCreatedAtLinkPath(@TempDir Path tmp) throws IOException {
var src = Files.createDirectories(tmp.resolve("src"));
var outside = Files.writeString(tmp.resolve("outside.txt"), "outside");
var link = src.resolve("link");
Files.createSymbolicLink(link, outside);
var bundle = tmp.resolve("sym.swb.zst");
BundleWriter.createBundle(bundle, List.of(link), c -> c.relativeRoot(src));
var dest = tmp.resolve("dest");
BundleReader.builder().build().extract(bundle, dest);
assertTrue(Files.isSymbolicLink(dest.resolve("link")),
"the symlink itself must be recreated at the link's path, not at its target");
}
@Test
public void testTruncatedEntryThrowsInsteadOfLooping(@TempDir Path tmp) throws IOException {
var bundle = tmp.resolve("truncated.swb.zst");
try (var zstd = new ZstdOutputStreamNoFinalizer(Files.newOutputStream(bundle), 3);
var out = new EntryOutputStream(zstd)) {
out.writeEntryHeader("short.bin", (short) 0, 100, System.currentTimeMillis());
out.write(new byte[10]); // 90 bytes missing
}
var dest = tmp.resolve("dest");
assertThrows(EOFException.class,
() -> BundleReader.builder().build().extract(bundle, dest));
}
}
@@ -0,0 +1,91 @@
package io.ib67.bundler;
import io.ib67.sfcraft.bundler.region.RegionFile;
import io.ib67.sfcraft.bundler.region.RegionParser;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.Unpooled;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
/**
* Simulates the full backup->restore cycle for region files: vanilla mca -> reassembled
* uncompressed (what goes into the bundle) -> reassembled ZLIB (what restore writes back),
* then verifies every chunk's uncompressed NBT is byte-identical and lives in the same
* header slot as in the original file.
*/
public class TestRegionRoundTrip {
@Test
public void roundTripPreservesEveryChunk() throws IOException {
roundTrip("region/r.9.-5.mca");
roundTrip("region/r.-1.-1.mca");
}
private static void roundTrip(String resource) throws IOException {
byte[] originalBytes;
try (var in = TestRegionRoundTrip.class.getClassLoader().getResourceAsStream(resource)) {
originalBytes = in.readAllBytes();
}
var alloc = ByteBufAllocator.DEFAULT;
// backup side: vanilla -> NONE bundle payload
ByteBuf intermediate;
try (var parser = new RegionParser(Unpooled.wrappedBuffer(originalBytes), alloc)) {
intermediate = parser.writeReassembled(RegionFile.CompressType.NONE);
}
// restore side: bundle payload -> ZLIB region written to disk
ByteBuf restored;
try (var parser = new RegionParser(intermediate, alloc)) {
restored = parser.writeReassembled(RegionFile.CompressType.ZLIB);
} finally {
intermediate.release();
}
try (var originalParser = new RegionParser(Unpooled.wrappedBuffer(originalBytes), alloc);
var restoredParser = new RegionParser(restored.readerIndex(0).retain(), alloc)) {
var originalChunks = collect(originalParser);
var restoredChunks = collect(restoredParser);
assertFalse(originalChunks.isEmpty(), resource + " should contain chunks");
assertEquals(originalChunks.keySet(), restoredChunks.keySet(),
"restored file must populate exactly the same chunk slots (no transposition)");
for (var entry : originalChunks.entrySet()) {
var origNbt = decompress(originalParser, entry.getValue(), alloc);
var restNbt = decompress(restoredParser, restoredChunks.get(entry.getKey()), alloc);
try {
assertEquals(origNbt, restNbt, "chunk " + entry.getKey() + " must survive the round trip unchanged");
} finally {
origNbt.release();
restNbt.release();
}
}
// timestamp table is copied verbatim
var origTimes = Unpooled.wrappedBuffer(originalBytes).slice(4096, 4096);
var restTimes = restored.slice(4096, 4096);
assertEquals(origTimes, restTimes, "chunk timestamps must survive the round trip");
} finally {
restored.release();
}
}
/** chunkXZ -> header entry */
private static Map<Short, Integer> collect(RegionParser parser) {
var result = new HashMap<Short, Integer>();
var chunks = parser.readAllValidChunks();
for (int i = 0; i < chunks.size(); i++) {
var key = chunks.getLong(i);
result.put(RegionParser.readChunkXZ(key), RegionParser.readEntry(key));
}
return result;
}
private static ByteBuf decompress(RegionParser parser, int entry, ByteBufAllocator alloc) throws IOException {
var buf = alloc.buffer();
parser.readChunkDecompress(entry, buf);
return buf;
}
}