overhaul thread model and backup reliability

Thread model:
  - replace the IOState CAS state machine with a StampedLock: region IO
    (RegionFileStorage.write/flush, now both guarded via @WrapMethod) holds
    the read lock, backups hold the write lock; the old witness-restore
    pattern could wedge the state permanently
  - run saveEverything on the main thread (was called from the backup
    executor), with shutdown-aware waiting; skip autoSave/saveEverything
    while a backup holds the write lock so the main thread never blocks
  - broadcast player messages from the main thread, honor `broadcast`

  Backup correctness:
  - watcher reported the parent directory instead of the changed file, so
    every incremental backup was empty; change sets are now deduplicated,
    atomically drained and merged back on failure; OVERFLOW forces a full
    backup; watch roots are re-registered when their inode changes (world
    format conversion orphaned the watches)
  - fix the region reassembly pipeline: zlib was "decompressed" with
    DeflaterInputStream, writeChunk dropped the 5-byte chunk header,
    the length field was off by one, coordinates were transposed, and
    sector padding was wrong on 4096-byte multiples — every reassembled
    chunk in old bundles is unrecoverable garbage
  - full backups now collect the file list after saving, off-thread

  New features:
  - reflink (FICLONE) staging via FFM: on CoW filesystems the write lock
    is only held while cloning, packing/uploading runs unlocked
  - chained restore: restoring an incremental automatically applies its
    full base and every incremental in between, in order
  - generation-based rotation implementing keepBackups; whole generations
    are deleted so an incremental never loses its base
  - Backup records carry subject/createdAt/incremental for chaining

  Reliability:
  - atomic index persistence (tmp + ATOMIC_MOVE), ConcurrentHashMap for
    the tracked-backup index, fsync bundles before storing
  - BundleReader: overlay extraction for chains, zip-slip guard, fix
    reversed createSymbolicLink args, EOF instead of infinite loop on
    truncated bundles
  - LocalBackupStrategy.deleteBackup compared the wrong field and could
    never delete; S3 retries now catch SdkException (retry was dead code),
    drop ifNoneMatch for idempotent retries
  - ordered shutdown: SERVER_STOPPING awaits in-flight backups before the
    final world save; SERVER_STOPPED runs restore, closes storage and
    cleans the temp dir with isolated error handling
  - fix /backup suspend being registered under /backup create

  Tests: region round-trip byte-comparison, BundleReader overlay/zip-slip/
  symlink/truncation, backup chain resolution and rotation.

  Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

build.gradle

@@ -71,6 +71,13 @@ dependencies {
     // https://mvnrepository.com/artifact/software.amazon.awssdk/s3
     shadow project(":bundler")
     shadow 'software.amazon.awssdk:s3:2.34.0'
+
+    testImplementation 'org.junit.jupiter:junit-jupiter:5.13.4'
+    testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
+}
+
+test {
+    useJUnitPlatform()
 }
 configurations.implementation.extendsFrom(configurations.shadow)
 shadowJar {

bundler/src/main/java/io/ib67/sfcraft/bundler/BundleReader.java

@@ -6,7 +6,9 @@ import io.ib67.sfcraft.bundler.region.RegionParser;
 import io.netty.buffer.ByteBufAllocator;
 import lombok.Builder;
 
+import java.io.EOFException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.channels.FileChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -32,8 +34,14 @@ public class BundleReader {
         this.dict = dict;
     }
 
+    /**
+     * Extracts a bundle into {@code destination}. Existing files are overwritten, which is
+     * what makes chained restores work: apply the full base first, then each incremental in
+     * order — later bundles win.
+     */
     public void extract(Path bundle, Path destination) throws IOException {
         var readMaps = new HashMap<String, Long>();
+        var destRoot = destination.toAbsolutePath().normalize();
         try (var zstd = new ZstdInputStreamNoFinalizer(Files.newInputStream(bundle, StandardOpenOption.READ));
              var in = new EntryInputStream(zstd)) {
             if (dict != null) zstd.setDict(dict);
@@ -42,62 +50,101 @@ public class BundleReader {
                 var time = entry.time();
                 if (time < minTime || time > maxTime) {
                     System.err.println("Skipping entry " + entry.name() + " (too old or too new)");
+                    skipExactly(in, entry.length());
                     continue;
                 }
                 var lastSeen = readMaps.get(entry.name());
                 if (lastSeen != null && skipOldEntry && entry.time() < lastSeen) {
-                    System.err.println("Skipping entry " + entry.name() + " (older than )");
+                    System.err.println("Skipping entry " + entry.name() + " (older than seen)");
+                    skipExactly(in, entry.length());
                     continue;
                 }
                 readMaps.put(entry.name(), time);
 
-                var path = destination.resolve(entry.name()); // todo prevent path underflow
+                var path = resolveSafely(destRoot, entry.name());
                 var parentDir = path.getParent();
                 if (Files.notExists(parentDir)) Files.createDirectories(parentDir);
                 if ((entry.attribute() & BundleEntry.ATTR_SYMLINK) != 0) {
-                    var toRead = (int) entry.length();
-                    var strArr = new byte[toRead];
-                    while ((toRead -= in.read(strArr, (int) (entry.length() - toRead), toRead)) > 0) ;
-                    var target = new String(strArr);
-                    var targetPath = Path.of(target);
-                    Files.createSymbolicLink(targetPath, path);
+                    var strArr = new byte[(int) entry.length()];
+                    readExactly(in, strArr);
+                    var targetPath = Path.of(new String(strArr));
+                    Files.deleteIfExists(path);
+                    Files.createSymbolicLink(path, targetPath);
                 } else if ((entry.attribute() & BundleEntry.ATTR_REASSEMBLE) != 0) {
                     var entryData = allocator.buffer();
-                    var toRead = entry.length();
-                    var buffer = this.buffer;
-                    while (toRead > 0) {
-                        var read = in.read(buffer, 0, (int) Math.min(buffer.length, toRead));
-                        toRead -= read;
-                        entryData.writeBytes(buffer, 0, read);
-                    }
-                    try (var reassembler = new RegionParser(entryData, allocator)) {
-                        var result = reassembler.writeReassembled(RegionFile.CompressType.ZLIB);
-                        try (var fc = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
-                            result.readBytes(fc, 0, result.readableBytes());
-                        } finally {
-                            result.release();
+                    try {
+                        var toRead = entry.length();
+                        var buffer = this.buffer;
+                        while (toRead > 0) {
+                            var read = in.read(buffer, 0, (int) Math.min(buffer.length, toRead));
+                            if (read < 0) throw new EOFException("Truncated bundle while reading " + entry.name());
+                            toRead -= read;
+                            entryData.writeBytes(buffer, 0, read);
                         }
-                    } catch (Exception e) {
-                        try (var ch = FileChannel.open(Path.of("dump.mca"), StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
-                            entryData.readerIndex(0);
-                            entryData.readBytes(ch, 0, entryData.readableBytes());
+                        try (var reassembler = new RegionParser(entryData, allocator)) {
+                            var result = reassembler.writeReassembled(RegionFile.CompressType.ZLIB);
+                            try (var fc = FileChannel.open(path, StandardOpenOption.CREATE,
+                                    StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) {
+                                result.readBytes(fc, 0, result.readableBytes());
+                            } finally {
+                                result.release();
+                            }
+                        } catch (Exception e) {
+                            var dump = Files.createTempFile("justbackup_dump_", ".mca");
+                            try (var ch = FileChannel.open(dump, StandardOpenOption.WRITE)) {
+                                entryData.readerIndex(0);
+                                entryData.readBytes(ch, 0, entryData.readableBytes());
+                            }
+                            throw new IOException("Error while reassembling " + entry.name() + ", dumped to " + dump, e);
                         }
-                        throw new IOException("Error while reassembling " + entry.name(), e);
+                    } finally {
+                        entryData.release();
                     }
                 } else {
                     writePlain(path, entry, in);
                 }
             }
+        }
+    }
 
+    /**
+     * Resolves an entry name inside the destination, refusing names that escape it
+     * (absolute paths or {@code ..} traversal).
+     */
+    private static Path resolveSafely(Path destRoot, String entryName) throws IOException {
+        var resolved = destRoot.resolve(entryName).normalize();
+        if (!resolved.startsWith(destRoot)) {
+            throw new IOException("Entry " + entryName + " escapes the destination directory, refusing to extract");
+        }
+        return resolved;
+    }
+
+    private static void readExactly(InputStream in, byte[] target) throws IOException {
+        int off = 0;
+        while (off < target.length) {
+            int read = in.read(target, off, target.length - off);
+            if (read < 0) throw new EOFException("Truncated bundle");
+            off += read;
+        }
+    }
+
+    private void skipExactly(InputStream in, long length) throws IOException {
+        var buffer = this.buffer;
+        while (length > 0) {
+            int read = in.read(buffer, 0, (int) Math.min(buffer.length, length));
+            if (read < 0) throw new EOFException("Truncated bundle");
+            length -= read;
         }
     }
 
     private void writePlain(Path path, BundleEntry entry, EntryInputStream in) throws IOException {
         var buffer = this.buffer;
-        try (var fc = Files.newOutputStream(path, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
+        try (var fc = Files.newOutputStream(path, StandardOpenOption.CREATE,
+                StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) {
             var toRead = entry.length();
             while (toRead > 0) {
                 var read = in.read(buffer, 0, (int) Math.min(buffer.length, toRead));
+                if (read < 0) throw new EOFException("Truncated bundle while reading " + entry.name());
                 toRead -= read;
                 fc.write(buffer, 0, read);
             }

bundler/src/main/java/io/ib67/sfcraft/bundler/BundleWriter.java

@@ -12,9 +12,12 @@ import lombok.Builder;
 import lombok.SneakyThrows;
 
 import java.io.*;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.*;
 import java.util.concurrent.*;
 import java.util.function.UnaryOperator;
@@ -131,14 +134,40 @@ public class BundleWriter implements Closeable {
             Iterable<Path> path,
             UnaryOperator<BundleWriterBuilder> config
     ) throws IOException {
-        try (var writer = config.apply(BundleWriter.builder())
-                .outputStream(Files.newOutputStream(pathToBundle))
-                .build()) {
-            for (var p : path) {
-                if (!Files.isRegularFile(p) || Files.size(p) == 0) continue;
-                var s = p.toString();
-                writer.write(p, (s.contains("region") && s.endsWith(".mca")) || s.endsWith(".dat"));
+        try (var channel = FileChannel.open(pathToBundle,
+                StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) {
+            try (var writer = config.apply(BundleWriter.builder())
+                    .outputStream(new NonClosingOutputStream(Channels.newOutputStream(channel)))
+                    .build()) {
+                for (var p : path) {
+                    if (!Files.isRegularFile(p) || Files.size(p) == 0) continue;
+                    var s = p.toString();
+                    writer.write(p, (s.contains("region") && s.endsWith(".mca")) || s.endsWith(".dat"));
+                }
             }
+            // the zstd epilogue is flushed by writer.close(); fsync before closing so a crash
+            // right after "backup success" cannot leave a hollow bundle behind
+            channel.force(true);
+        }
+    }
+
+    /**
+     * Lets the zstd stream be closed (writing its epilogue) without closing the underlying
+     * channel, so the caller can fsync it afterwards.
+     */
+    private static final class NonClosingOutputStream extends FilterOutputStream {
+        NonClosingOutputStream(OutputStream out) {
+            super(out);
+        }
+
+        @Override
+        public void write(byte[] b, int off, int len) throws IOException {
+            out.write(b, off, len);
+        }
+
+        @Override
+        public void close() throws IOException {
+            out.flush();
         }
     }
 
@@ -191,8 +220,10 @@ public class BundleWriter implements Closeable {
             result.readBytes(out, length);
             result.release();
             // all done
-        } catch (IOException e) {
-            System.err.println("Failed to parse region " + path + ": " + e);
+        } catch (Exception e) {
+            // any parse failure (not only IOException) must fall back to a plain copy instead
+            // of failing the whole backup
+            System.err.println("Failed to parse region " + path + ", storing it as-is: " + e);
             finalResult.release();
             writePlain(path);
             return;
@@ -205,7 +236,8 @@ public class BundleWriter implements Closeable {
     private void writePlain(Path path) throws IOException {
         var rPath = relativeRoot.toAbsolutePath().relativize(path.toAbsolutePath());
         var outputStream = this.outputStream;
-        outputStream.writeEntryHeader(rPath.toString(), (short) 0, Files.size(path), 0);
+        outputStream.writeEntryHeader(rPath.toString(), (short) 0, Files.size(path),
+                Files.getLastModifiedTime(path).toMillis());
         var read = 0;
         try (var fs = Files.newInputStream(path)) {
             var buffer = BundleWriter.buffer.get();

bundler/src/main/java/io/ib67/sfcraft/bundler/region/RegionFile.java

@@ -45,31 +45,31 @@ public class RegionFile implements Closeable {
         return (short) ((z & 0b111111) << 6 | (x & 0b111111));
     }
 
-    // this retains the input bytebuf.
     public void writeChunk(int chunkX, int chunkZ, CompressType type, ByteBuf data) throws IOException {
-        if (chunkX > 32 || chunkZ > 32 || chunkX < 0 || chunkZ < 0)
+        if (chunkX >= 32 || chunkZ >= 32 || chunkX < 0 || chunkZ < 0)
             throw new IllegalArgumentException("Invalid chunkX+chunkZ: " + chunkX + " " + chunkZ);
         var key = getChunkKey(chunkX, chunkZ);
-        var buf = allocator.buffer(5 + data.readableBytes());
+        ByteBuf payload;
         if (type == CompressType.NONE) {
-            buf.writeInt(data.readableBytes());
-            buf.writeByte(type.index);
-            buf.writeBytes(data);
+            payload = allocator.buffer(data.readableBytes());
+            payload.writeBytes(data);
         } else {
-            try (var out = findCompressor(type, new ByteBufOutputStream(buf))) {
-                while (data.readableBytes() > 0) data.readBytes(out, data.readableBytes());
-                var head = allocator.buffer(5);
-                head.writeInt(buf.readableBytes());
-                head.writeByte(type.index);
-                var composite =  allocator.compositeBuffer();
-                composite.addComponent(true, head);
-                composite.addComponent(true, buf);
+            payload = allocator.buffer();
+            try (var out = findCompressor(type, new ByteBufOutputStream(payload))) {
+                data.readBytes(out, data.readableBytes());
             } catch (IOException e) {
-                buf.release();
+                payload.release();
                 throw e;
             }
         }
-        regionData.put(key, buf);
+        var buf = allocator.buffer(5 + payload.readableBytes());
+        // vanilla region format: the length field also counts the compression type byte
+        buf.writeInt(payload.readableBytes() + 1);
+        buf.writeByte(type.index);
+        buf.writeBytes(payload);
+        payload.release();
+        var previous = regionData.put(key, buf);
+        if (previous != null) previous.release();
     }
 
     private OutputStream findCompressor(CompressType type, OutputStream out) throws IOException {
@@ -82,7 +82,7 @@ public class RegionFile implements Closeable {
     }
 
     public void updateChunkTime(int chunkX, int chunkZ, int timestamp) {
-        if (chunkX > 32 || chunkZ > 32 || chunkX < 0 || chunkZ < 0)
+        if (chunkX >= 32 || chunkZ >= 32 || chunkX < 0 || chunkZ < 0)
             throw new IllegalArgumentException("Invalid chunkX+chunkZ: " + chunkX + " " + chunkZ);
         var index = (chunkX + 32 * chunkZ) * 4;
         this.timestamp.setInt(index, timestamp);
@@ -105,7 +105,7 @@ public class RegionFile implements Closeable {
             var chunk = entry.getValue();
             var size = chunk.readableBytes();
             var sizeSectors = Math.ceilDiv(size, 4096);
-            var zeros = 4096 - (size % 4096);
+            var zeros = (4096 - (size % 4096)) % 4096;
             for (int i = 0; i < zeros; i++) {
                 chunk.writeByte(0);
             }

bundler/src/main/java/io/ib67/sfcraft/bundler/region/RegionParser.java

@@ -10,8 +10,8 @@ import java.nio.channels.FileChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
-import java.util.zip.DeflaterInputStream;
 import java.util.zip.GZIPInputStream;
+import java.util.zip.InflaterInputStream;
 
 public class RegionParser implements Closeable {
     protected final ByteBuf source;
@@ -28,7 +28,9 @@ public class RegionParser implements Closeable {
         try (var fc = FileChannel.open(file, StandardOpenOption.READ)) {
             var toRead = size;
             while (toRead > 0) {
-                toRead -= source.writeBytes(fc, 0L, (int) size);
+                var read = source.writeBytes(fc, (long) (size - toRead), toRead);
+                if (read < 0) throw new EOFException("Region file " + file + " shrank while reading");
+                toRead -= read;
             }
         }
     }
@@ -98,8 +100,9 @@ public class RegionParser implements Closeable {
                 var key = chunks.getLong(i);
                 var chunkXZ = readChunkXZ(key);
                 var entry = readEntry(key);
-                short x = (short) (chunkXZ & 0x3F);
-                short z = (short) ((chunkXZ >>> 6) & 0x3F);
+                // packed as (x << 6) | z in readAllValidChunks
+                short x = (short) ((chunkXZ >>> 6) & 0x3F);
+                short z = (short) (chunkXZ & 0x3F);
                 var compressType = readChunk(entry, compressed);
                 try (var decompressor = getDecompressorByType(compressType, compressed)) {
                     while (uncompressed.writeBytes(decompressor, 4096) != -1) ;
@@ -116,7 +119,7 @@ public class RegionParser implements Closeable {
     private InputStream getDecompressorByType(int compressionType, ByteBuf in) throws IOException {
         return switch (compressionType) {
             case 1 -> new GZIPInputStream(new ByteBufInputStream(in));
-            case 2 -> new DeflaterInputStream(new ByteBufInputStream(in));
+            case 2 -> new InflaterInputStream(new ByteBufInputStream(in));
             case 3 -> new ByteBufInputStream(in);
             case 4 -> new LZ4BlockInputStream(new ByteBufInputStream(in));
             // user defined compression algorithm, not going to support.

bundler/src/test/java/io/ib67/bundler/TestBundleReader.java

@@ -0,0 +1,88 @@
+package io.ib67.bundler;
+
+import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
+import io.ib67.sfcraft.bundler.BundleReader;
+import io.ib67.sfcraft.bundler.BundleWriter;
+import io.ib67.sfcraft.bundler.EntryOutputStream;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class TestBundleReader {
+
+    @Test
+    public void testChainedOverlayRestore(@TempDir Path tmp) throws IOException {
+        var src = Files.createDirectories(tmp.resolve("src"));
+        Files.writeString(src.resolve("a.txt"), "version-1-that-is-long");
+        Files.writeString(src.resolve("b.txt"), "untouched");
+        var fullBundle = tmp.resolve("full.swb.zst");
+        BundleWriter.createBundle(fullBundle, List.of(src.resolve("a.txt"), src.resolve("b.txt")),
+                c -> c.relativeRoot(src));
+
+        // shorter content verifies the overlay truncates instead of leaving a stale tail
+        Files.writeString(src.resolve("a.txt"), "v2");
+        var incrBundle = tmp.resolve("incr.swb.zst");
+        BundleWriter.createBundle(incrBundle, List.of(src.resolve("a.txt")),
+                c -> c.relativeRoot(src));
+
+        var dest = tmp.resolve("dest");
+        var reader = BundleReader.builder().build();
+        reader.extract(fullBundle, dest);
+        reader.extract(incrBundle, dest);
+
+        assertEquals("v2", Files.readString(dest.resolve("a.txt")));
+        assertEquals("untouched", Files.readString(dest.resolve("b.txt")));
+    }
+
+    @Test
+    public void testZipSlipIsRejected(@TempDir Path tmp) throws IOException {
+        var bundle = tmp.resolve("evil.swb.zst");
+        var payload = "owned".getBytes();
+        try (var zstd = new ZstdOutputStreamNoFinalizer(Files.newOutputStream(bundle), 3);
+             var out = new EntryOutputStream(zstd)) {
+            out.writeEntryHeader("../evil.txt", (short) 0, payload.length, System.currentTimeMillis());
+            out.write(payload);
+        }
+        var dest = tmp.resolve("dest");
+        var ex = assertThrows(IOException.class,
+                () -> BundleReader.builder().build().extract(bundle, dest));
+        assertTrue(ex.getMessage().contains("escapes"), ex.getMessage());
+        assertFalse(Files.exists(tmp.resolve("evil.txt")));
+    }
+
+    @Test
+    public void testSymlinkIsCreatedAtLinkPath(@TempDir Path tmp) throws IOException {
+        var src = Files.createDirectories(tmp.resolve("src"));
+        var outside = Files.writeString(tmp.resolve("outside.txt"), "outside");
+        var link = src.resolve("link");
+        Files.createSymbolicLink(link, outside);
+
+        var bundle = tmp.resolve("sym.swb.zst");
+        BundleWriter.createBundle(bundle, List.of(link), c -> c.relativeRoot(src));
+
+        var dest = tmp.resolve("dest");
+        BundleReader.builder().build().extract(bundle, dest);
+        assertTrue(Files.isSymbolicLink(dest.resolve("link")),
+                "the symlink itself must be recreated at the link's path, not at its target");
+    }
+
+    @Test
+    public void testTruncatedEntryThrowsInsteadOfLooping(@TempDir Path tmp) throws IOException {
+        var bundle = tmp.resolve("truncated.swb.zst");
+        try (var zstd = new ZstdOutputStreamNoFinalizer(Files.newOutputStream(bundle), 3);
+             var out = new EntryOutputStream(zstd)) {
+            out.writeEntryHeader("short.bin", (short) 0, 100, System.currentTimeMillis());
+            out.write(new byte[10]); // 90 bytes missing
+        }
+        var dest = tmp.resolve("dest");
+        assertThrows(EOFException.class,
+                () -> BundleReader.builder().build().extract(bundle, dest));
+    }
+}

bundler/src/test/java/io/ib67/bundler/TestRegionRoundTrip.java

@@ -0,0 +1,91 @@
+package io.ib67.bundler;
+
+import io.ib67.sfcraft.bundler.region.RegionFile;
+import io.ib67.sfcraft.bundler.region.RegionParser;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.Unpooled;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * Simulates the full backup->restore cycle for region files: vanilla mca -> reassembled
+ * uncompressed (what goes into the bundle) -> reassembled ZLIB (what restore writes back),
+ * then verifies every chunk's uncompressed NBT is byte-identical and lives in the same
+ * header slot as in the original file.
+ */
+public class TestRegionRoundTrip {
+    @Test
+    public void roundTripPreservesEveryChunk() throws IOException {
+        roundTrip("region/r.9.-5.mca");
+        roundTrip("region/r.-1.-1.mca");
+    }
+
+    private static void roundTrip(String resource) throws IOException {
+        byte[] originalBytes;
+        try (var in = TestRegionRoundTrip.class.getClassLoader().getResourceAsStream(resource)) {
+            originalBytes = in.readAllBytes();
+        }
+        var alloc = ByteBufAllocator.DEFAULT;
+
+        // backup side: vanilla -> NONE bundle payload
+        ByteBuf intermediate;
+        try (var parser = new RegionParser(Unpooled.wrappedBuffer(originalBytes), alloc)) {
+            intermediate = parser.writeReassembled(RegionFile.CompressType.NONE);
+        }
+        // restore side: bundle payload -> ZLIB region written to disk
+        ByteBuf restored;
+        try (var parser = new RegionParser(intermediate, alloc)) {
+            restored = parser.writeReassembled(RegionFile.CompressType.ZLIB);
+        } finally {
+            intermediate.release();
+        }
+
+        try (var originalParser = new RegionParser(Unpooled.wrappedBuffer(originalBytes), alloc);
+             var restoredParser = new RegionParser(restored.readerIndex(0).retain(), alloc)) {
+            var originalChunks = collect(originalParser);
+            var restoredChunks = collect(restoredParser);
+            assertFalse(originalChunks.isEmpty(), resource + " should contain chunks");
+            assertEquals(originalChunks.keySet(), restoredChunks.keySet(),
+                    "restored file must populate exactly the same chunk slots (no transposition)");
+            for (var entry : originalChunks.entrySet()) {
+                var origNbt = decompress(originalParser, entry.getValue(), alloc);
+                var restNbt = decompress(restoredParser, restoredChunks.get(entry.getKey()), alloc);
+                try {
+                    assertEquals(origNbt, restNbt, "chunk " + entry.getKey() + " must survive the round trip unchanged");
+                } finally {
+                    origNbt.release();
+                    restNbt.release();
+                }
+            }
+            // timestamp table is copied verbatim
+            var origTimes = Unpooled.wrappedBuffer(originalBytes).slice(4096, 4096);
+            var restTimes = restored.slice(4096, 4096);
+            assertEquals(origTimes, restTimes, "chunk timestamps must survive the round trip");
+        } finally {
+            restored.release();
+        }
+    }
+
+    /** chunkXZ -> header entry */
+    private static Map<Short, Integer> collect(RegionParser parser) {
+        var result = new HashMap<Short, Integer>();
+        var chunks = parser.readAllValidChunks();
+        for (int i = 0; i < chunks.size(); i++) {
+            var key = chunks.getLong(i);
+            result.put(RegionParser.readChunkXZ(key), RegionParser.readEntry(key));
+        }
+        return result;
+    }
+
+    private static ByteBuf decompress(RegionParser parser, int entry, ByteBufAllocator alloc) throws IOException {
+        var buf = alloc.buffer();
+        parser.readChunkDecompress(entry, buf);
+        return buf;
+    }
+}

src/main/java/io/ib67/sfcraft/Backup.java

@@ -2,17 +2,41 @@ package io.ib67.sfcraft;
 
 import net.minecraft.network.chat.Component;
 
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+
+/**
+ * @param subject   name of the backup subject this bundle belongs to; null on entries created
+ *                  by older versions (those are treated as standalone full backups)
+ * @param createdAt epoch millis of creation; 0 on legacy entries
+ */
 public record Backup(
         String name,
         String backupKey,
         String type,
         String from,
+        String subject,
         boolean incremental,
+        long createdAt,
         long sizeTotal
 ) {
-    public Component toText(){
-        return Component.nullToEmpty("sizeTotal: "+Math.round(((sizeTotal / 1024.0) / 1024.0) * 100) * 0.01 + "MiB, "+
-                (incremental ? "incremental" :"") + ", type: "+type+"."
-        );
+    private static final DateTimeFormatter TIME_FORMAT =
+            DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneId.systemDefault());
+
+    public boolean legacy() {
+        return subject == null || createdAt <= 0;
+    }
+
+    public Component toText() {
+        var size = Math.round(((sizeTotal / 1024.0) / 1024.0) * 100) * 0.01;
+        var sb = new StringBuilder()
+                .append(incremental ? "INCR" : "FULL")
+                .append(", ").append(size).append("MiB")
+                .append(", type: ").append(type);
+        if (createdAt > 0) {
+            sb.append(", at ").append(TIME_FORMAT.format(Instant.ofEpochMilli(createdAt)));
+        }
+        return Component.nullToEmpty(sb.toString());
     }
 }

src/main/java/io/ib67/sfcraft/BackupCommands.java

@@ -7,19 +7,20 @@ import com.mojang.brigadier.arguments.StringArgumentType;
 import com.mojang.brigadier.context.CommandContext;
 import com.mojang.brigadier.suggestion.Suggestions;
 import com.mojang.brigadier.suggestion.SuggestionsBuilder;
+import io.ib67.sfcraft.strategy.BackupChains;
 import lombok.extern.log4j.Log4j2;
 import net.minecraft.commands.CommandBuildContext;
 import net.minecraft.commands.CommandSourceStack;
 import net.minecraft.commands.Commands;
 import net.minecraft.network.chat.ClickEvent;
 import net.minecraft.network.chat.Component;
+
 import java.awt.*;
 import java.io.IOException;
 import java.lang.ref.WeakReference;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 
 import static net.minecraft.commands.Commands.argument;
@@ -28,7 +29,7 @@ import static net.minecraft.commands.Commands.literal;
 @Log4j2
 public class BackupCommands {
     protected final JustBackupMod mod;
-    protected Runnable restoreIssued;
+    protected volatile Runnable restoreIssued;
     protected final List<String> helpMessage = List.of(
             "__ Help messages of JustBackup __",
             " /backup help ",
@@ -37,6 +38,8 @@ public class BackupCommands {
             "    -- List backups",
             " /backup restore <backupKey>",
             "    -- Restore the backup on next server restart.",
+            "    Restoring an incremental backup automatically applies its full base",
+            "    and every incremental in between, in order.",
             "    Can be cancelled by /backup cancelrestore",
             " /backup delete <backupKey>",
             "    -- Delete a backup. This does not ask for a confirmation.",
@@ -62,6 +65,7 @@ public class BackupCommands {
                 .then(literal("help").executes(this::cmdHelp))
                 .then(literal("list").executes(this::cmdList))
                 .then(literal("cancelrestore").executes(this::cmdCancelRestore))
+                .then(literal("suspend").executes(this::cmdSuspend))
                 .then(literal("delete")
                         .then(argument("backupKey", StringArgumentType.greedyString())
                                 .suggests(this::suggestBackups)
@@ -74,9 +78,7 @@ public class BackupCommands {
                         .then(argument("incremental", BoolArgumentType.bool())
                                 .then(argument("subject", StringArgumentType.greedyString())
                                         .executes(this::createBackupBySubject))
-                                .then(literal("@all").executes(this::fullBackup)))
-                        .then(literal("suspend").executes(this::cmdSuspend))
-                ));
+                                .then(literal("@all").executes(this::fullBackup)))));
     }
 
     private int createBackupBySubject(CommandContext<CommandSourceStack> context) {
@@ -94,18 +96,23 @@ public class BackupCommands {
         s.sendSystemMessage(Component.literal("The issued backup has been scheduled.").withColor(Color.CYAN.getRGB()));
         var weakRef = new WeakReference<>(s);
         mod.backupAll(incremental).thenAccept(result -> {
-            var ref = weakRef.get();
-            if (ref == null) {
-                ref = mod.server.createCommandSourceStack();
-            }
-            ref.sendSystemMessage(Component.literal("-- BACKUP RESULT SUMMARY --"));
-            for (var entry : result.entrySet()) {
-                var k = entry.getKey();
-                var v = entry.getValue();
-                ref.sendSystemMessage(Component.literal("  - [" + k + "]: ").append(v.toText()));
-                ref.sendSystemMessage(Component.literal("BACKUP KEY: " + v.backupKey()).withColor(Color.GRAY.getRGB()));
-                ref.sendSystemMessage(Component.literal("GENERATED FROM: " + v.from()).withColor(Color.GRAY.getRGB()));
-            }
+            var server = mod.server;
+            if (server == null || !server.isRunning()) return;
+            // completion runs on the backup executor; command sources must be used on the main thread
+            server.submit(() -> {
+                var ref = weakRef.get();
+                if (ref == null) {
+                    ref = server.createCommandSourceStack();
+                }
+                ref.sendSystemMessage(Component.literal("-- BACKUP RESULT SUMMARY --"));
+                for (var entry : result.entrySet()) {
+                    var k = entry.getKey();
+                    var v = entry.getValue();
+                    ref.sendSystemMessage(Component.literal("  - [" + k + "]: ").append(v.toText()));
+                    ref.sendSystemMessage(Component.literal("BACKUP KEY: " + v.backupKey()).withColor(Color.GRAY.getRGB()));
+                    ref.sendSystemMessage(Component.literal("GENERATED FROM: " + v.from()).withColor(Color.GRAY.getRGB()));
+                }
+            });
         });
         return Command.SINGLE_SUCCESS;
     }
@@ -136,31 +143,51 @@ public class BackupCommands {
             s.sendSystemMessage(Component.literal("Invalid backup " + backupKey + ". Note: Use backupKey instead of backupName!"));
             return 0;
         }
+        final List<Backup> chain;
+        try {
+            chain = BackupChains.resolveChain(mod.tracker.getTrackedBackups().values(), backup);
+        } catch (IllegalStateException e) {
+            s.sendSystemMessage(Component.literal(e.getMessage()).withColor(Color.RED.getRGB()));
+            return 0;
+        }
+        s.sendSystemMessage(Component.literal("The following " + chain.size() + " bundle(s) will be applied in order:"));
+        for (var b : chain) {
+            s.sendSystemMessage(Component.literal("  -> [" + (b.incremental() ? "INCR" : "FULL") + "] " + b.backupKey())
+                    .withColor(Color.GRAY.getRGB()));
+        }
         restoreIssued = () -> {
-            log.warn("EXTRACTING BACKUP " + backup + " TO " + backup.from());
-            log.warn(" == STEP 1 == Make a backup for the destination.");
             var destination = Path.of(backup.from());
-            var movedDst = destination.resolveSibling(destination.getFileName().toString()+"_bak_"+System.currentTimeMillis());
-            try{
+            var movedDst = destination.resolveSibling(destination.getFileName().toString() + "_bak_" + System.currentTimeMillis());
+            log.warn("EXTRACTING {} BUNDLE(S) TO {}", chain.size(), destination);
+            log.warn(" == STEP 1 == Make a backup for the destination.");
+            try {
                 Files.move(destination, movedDst);
                 log.info("{} has been moved to {}", destination, movedDst);
             } catch (IOException e) {
-                log.error(e);
-                log.error("CANNOT MOVE "+destination+" TO "+movedDst);
-                log.error("BACKUP INTERRUPTED. Here are some tips helping you out:");
+                log.error("CANNOT MOVE {} TO {}", destination, movedDst, e);
+                log.error("RESTORE INTERRUPTED. Here are some tips helping you out:");
                 log.error(" -- A. Merge already moved files");
                 log.error("  Try this command in your server directory: (linux)");
-                log.error("  $ cp ./"+movedDst+"/* ./"+destination);
+                log.error("  $ cp ./{}/* ./{}", movedDst, destination);
                 log.error(" -- B. Remove them all and use external unbundler tools");
                 log.error("  You may want to copy them elsewhere first (see kind A)");
-                log.error("  $ rm -r ./"+destination+" ./"+movedDst);
+                log.error("  $ rm -r ./{} ./{}", destination, movedDst);
                 log.error("  Visit https://github.com/saltedfishclub/justbackup for the usage of bundler tools");
                 log.error(" ------- ERROR END -------");
                 return;
             }
-            log.info(" == STEP 2 == Recovering backup "+backup);
-            mod.tracker.recoverBackup(backup, destination);
-            log.info("Backup has been recovered.");
+            log.info(" == STEP 2 == Applying {} bundle(s)", chain.size());
+            try {
+                for (int i = 0; i < chain.size(); i++) {
+                    var b = chain.get(i);
+                    log.info("Applying {} ({}/{})", b.backupKey(), i + 1, chain.size());
+                    mod.tracker.recoverBackup(b, destination);
+                }
+                log.info("Backup has been recovered.");
+            } catch (Exception e) {
+                log.error("RESTORE FAILED midway. Your original world is intact at {}", movedDst, e);
+                log.error("Move it back to {} to undo the restore.", destination);
+            }
         };
         s.sendSystemMessage(Component.literal("Backup restoration task has been scheduled!"));
         s.sendSystemMessage(Component.literal("Restart your server to take changes."));
@@ -180,7 +207,7 @@ public class BackupCommands {
             source.sendSystemMessage(Component.nullToEmpty("Backup has been deleted."));
         } catch (Exception e) {
             source.sendSystemMessage(Component.nullToEmpty("Backup has not been deleted successfully. ERROR: " + e));
-            log.error(e);
+            log.error("Cannot delete backup {}", backupKey, e);
         }
         return Command.SINGLE_SUCCESS;
     }

src/main/java/io/ib67/sfcraft/BackupSubject.java

@@ -1,11 +1,64 @@
 package io.ib67.sfcraft;
 
 import java.nio.file.Path;
-import java.util.List;
+import java.util.HashSet;
+import java.util.Set;
 
-public record BackupSubject(
-        String name,
-        Path root,
-        List<Path> changedFiles
-) {
+/**
+ * A directory tree being backed up, plus the set of files the watcher saw change since the
+ * last incremental backup. All access to the change set goes through synchronized methods so
+ * the watcher's adds and the worker's drain can never interleave badly (no lost events, no
+ * ConcurrentModificationException while bundling).
+ */
+public final class BackupSubject {
+    private final String name;
+    private final Path root;
+    private final Set<Path> changedFiles = new HashSet<>();
+    private boolean forceFullNext;
+
+    public BackupSubject(String name, Path root) {
+        this.name = name;
+        this.root = root;
+    }
+
+    public String name() {
+        return name;
+    }
+
+    public Path root() {
+        return root;
+    }
+
+    public synchronized void addChanged(Path file) {
+        changedFiles.add(file);
+    }
+
+    /**
+     * Atomically takes ownership of the accumulated change set, leaving an empty one behind.
+     * If the backup fails, give the files back via {@link #mergeBack(Set)}.
+     */
+    public synchronized Set<Path> drainChanged() {
+        var drained = new HashSet<>(changedFiles);
+        changedFiles.clear();
+        return drained;
+    }
+
+    public synchronized void mergeBack(Set<Path> files) {
+        changedFiles.addAll(files);
+    }
+
+    /**
+     * Marks the next backup as full, used when the watcher reports an OVERFLOW (events were
+     * dropped by the OS, so the change set can no longer be trusted).
+     */
+    public synchronized void forceFullNext() {
+        forceFullNext = true;
+    }
+
+    public synchronized boolean consumeForceFull() {
+        var v = forceFullNext;
+        forceFullNext = false;
+        if (v) changedFiles.clear();
+        return v;
+    }
 }

src/main/java/io/ib67/sfcraft/BackupWorker.java

@@ -6,21 +6,45 @@ import lombok.SneakyThrows;
 import lombok.extern.log4j.Log4j2;
 import net.minecraft.server.MinecraftServer;
 
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
+import java.util.stream.Stream;
 
-import static io.ib67.sfcraft.Globals.BACKUP_LOCK;
+import static io.ib67.sfcraft.Globals.WORLD_IO_LOCK;
 
+/**
+ * Runs on the backup executor thread. Flow:
+ * <ol>
+ *   <li>Save the world on the main thread (saveEverything is not thread-safe anywhere else)
+ *       with flush=true, draining the IOWorkers so region files are complete on disk.</li>
+ *   <li>Acquire the write side of {@link Globals#WORLD_IO_LOCK}, parking all region IO.</li>
+ *   <li>Reflink-clone the files to a staging dir and release the lock early (CoW
+ *       filesystems), or keep holding it while packing (fallback).</li>
+ *   <li>Hand the bundle to the storage strategy.</li>
+ * </ol>
+ * The main thread never blocks on the lock — autoSave/saveEverything are skipped while it is
+ * write-locked (see MixinMinecraftServer) — so waiting for main-thread tasks while holding
+ * the write lock cannot deadlock.
+ */
 @Log4j2
 public record BackupWorker(
         MinecraftServer server,
         BackupStrategy strategy,
         Path tmpDir,
         BackupSubject subject,
+        boolean incremental,
+        List<Path> excludedRoots,
         UnaryOperator<BundleWriter.BundleWriterBuilder> config
 ) implements Supplier<Backup> {
+    private static final long LOCK_TIMEOUT_SECONDS = 60;
 
     @SneakyThrows
     public BackupWorker {
@@ -33,30 +57,116 @@ public record BackupWorker(
         if (server == null) {
             throw new IllegalStateException("MinecraftServer is not initialized yet, not doing backup");
         }
-        server.saveEverything(true, true, false);
-        while (!BACKUP_LOCK.compareAndSet(IOState.IDLE, IOState.BACKUP)) {
-            log.info("Server is saving world while performing backup! waiting...");
-            Thread.sleep(2000);
+        var createdAt = System.currentTimeMillis();
+        var full = !incremental || subject.consumeForceFull();
+
+        runOnMainThread(() -> server.saveEverything(true, true, false), "world save");
+
+        long stamp = WORLD_IO_LOCK.tryWriteLock(LOCK_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+        if (stamp == 0) {
+            throw new IllegalStateException("World IO is still busy after " + LOCK_TIMEOUT_SECONDS
+                    + "s, aborting backup of " + subject.name());
         }
-        // performing backup is now true
-        var tmpBundle = tmpDir.resolve("backup_" + subject.name() + "_" + System.currentTimeMillis() + ".swb.zst");
+
+        var tmpBundle = tmpDir.resolve("backup_" + subject.name() + "_" + createdAt + ".swb.zst");
+        Set<Path> drained = null;
+        Path stagingRoot = null;
         try {
-            // tmp file path
-            log.info("Creating bundle {}", tmpBundle);
-            BundleWriter.createBundle(tmpBundle, subject.changedFiles(), config);
+            // Barrier: a main-thread save that started before we locked has finished by now.
+            runOnMainThread(() -> {
+            }, "main-thread barrier");
+
+            List<Path> files;
+            if (full) {
+                files = collectFullFileList();
+            } else {
+                drained = subject.drainChanged();
+                files = drained.stream().filter(this::eligible).toList();
+            }
+
+            var packRoot = subject.root();
+            stagingRoot = ReflinkCloner.tryStage(subject.name(), packRoot, files,
+                    tmpDir.resolve("staging").resolve(subject.name() + "_" + createdAt));
+            if (stagingRoot != null) {
+                var absRoot = packRoot.toAbsolutePath().normalize();
+                var absStaging = stagingRoot;
+                files = files.stream()
+                        .map(f -> absStaging.resolve(absRoot.relativize(f.toAbsolutePath().normalize())))
+                        .toList();
+                packRoot = stagingRoot;
+                WORLD_IO_LOCK.unlockWrite(stamp);
+                stamp = 0;
+                log.info("Reflinked {} files to staging; world IO resumed, packing without the lock.", files.size());
+            }
+
+            log.info("Creating {} bundle {} ({} files)", full ? "full" : "incremental", tmpBundle, files.size());
+            var finalPackRoot = packRoot;
+            BundleWriter.createBundle(tmpBundle, files, c -> config.apply(c).relativeRoot(finalPackRoot));
         } catch (Exception e) {
+            if (drained != null) subject.mergeBack(drained);
+            Files.deleteIfExists(tmpBundle);
             throw e;
         } finally {
-            // Though we won't access the state within backup progress, there are certain cases
-            // that this state will be changed to STORAGE_SYNC.
-            // Also take a look at MixinRegionBasedStorage.
-            while (!BACKUP_LOCK.compareAndSet(IOState.BACKUP, IOState.IDLE)) {
-                log.error("Expect BACKUP state but got {}", BACKUP_LOCK.get());
-                Thread.sleep(2000);
+            if (stamp != 0) WORLD_IO_LOCK.unlockWrite(stamp);
+            if (stagingRoot != null) {
+                try {
+                    FsUtil.deleteRecursively(stagingRoot);
+                } catch (IOException e) {
+                    log.warn("Cannot clean up staging dir {}", stagingRoot, e);
+                }
             }
         }
-        var result = strategy.createBackup(subject.name(), subject.root(), tmpBundle);
-        Files.deleteIfExists(tmpBundle);
-        return result;
+
+        try {
+            return strategy.createBackup(subject.name(), subject.root(), tmpBundle, !full, createdAt);
+        } catch (Exception e) {
+            if (drained != null) subject.mergeBack(drained);
+            throw e;
+        } finally {
+            Files.deleteIfExists(tmpBundle);
+        }
+    }
+
+    /**
+     * Runs a task on the main thread, but never hangs: once the server leaves its tick loop
+     * (shutdown) submitted tasks are no longer processed, so we poll and abort the backup
+     * instead of blocking forever.
+     */
+    private void runOnMainThread(Runnable task, String what) {
+        var future = server.submit(task);
+        while (true) {
+            try {
+                future.get(1, TimeUnit.SECONDS);
+                return;
+            } catch (TimeoutException e) {
+                if (!server.isRunning()) {
+                    future.cancel(false);
+                    throw new IllegalStateException("Server is shutting down, aborting backup while waiting for " + what);
+                }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new IllegalStateException("Interrupted while waiting for " + what);
+            } catch (ExecutionException e) {
+                throw new IllegalStateException(what + " failed", e.getCause());
+            }
+        }
+    }
+
+    @SneakyThrows
+    private List<Path> collectFullFileList() {
+        try (Stream<Path> s = Files.walk(subject.root())) {
+            return s.filter(Files::isRegularFile)
+                    .filter(this::eligible)
+                    .toList();
+        }
+    }
+
+    private boolean eligible(Path path) {
+        if (path.getFileName().toString().equals("session.lock")) return false;
+        var abs = path.toAbsolutePath().normalize();
+        for (Path excluded : excludedRoots) {
+            if (abs.startsWith(excluded.toAbsolutePath().normalize())) return false;
+        }
+        return true;
     }
 }

src/main/java/io/ib67/sfcraft/FileWatcherThread.java

@@ -3,87 +3,181 @@ package io.ib67.sfcraft;
 import lombok.SneakyThrows;
 import lombok.extern.log4j.Log4j2;
 
+import java.io.IOException;
 import java.nio.file.*;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Objects;
 import java.util.concurrent.TimeUnit;
-import java.util.function.BiConsumer;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 @Log4j2
 public class FileWatcherThread extends Thread {
-    protected final Map<Path, String> watchSubject;
-    protected final BiConsumer<String, Path> eventListener;
+    private static final long VALIDATE_INTERVAL_MILLIS = 10_000;
 
-    public FileWatcherThread(
-            Map<String, Path> watchSubject,
-            BiConsumer<String, Path> eventListener
-    ) {
-        this.watchSubject = watchSubject.entrySet()
-                .stream()
-                // todo support for nonexist dir
-                .flatMap(it -> walkThrough(it.getValue()).map(r -> Map.entry(r.toAbsolutePath(), it.getKey())))
-                .filter(it->Files.isDirectory(it.getKey()))
-                .collect(Collectors.toMap(
-                        Map.Entry::getKey,
-                        Map.Entry::getValue
-                ));
-        this.eventListener = eventListener;
-        setName("JustBackup/FileWatcher");
-        setDaemon(true);
+    public interface Listener {
+        void onChange(String subject, Path file);
+
+        /**
+         * Events for this subject were lost (OS queue overflow, or its directory tree was
+         * replaced underneath us); the accumulated change set can no longer be trusted and
+         * the next backup should be a full one.
+         */
+        void onOverflow(String subject);
     }
 
+    /** subject name -> root directory (absolute) */
+    protected final Map<String, Path> subjectRoots = new HashMap<>();
+    /** subject name -> fileKey (inode) of the root at registration time */
+    protected final Map<String, Object> subjectRootKeys = new HashMap<>();
+    /** watched directory (absolute) -> subject name */
+    protected final Map<Path, String> watchedDirs = new HashMap<>();
+    protected final Map<WatchKey, Path> keyToDir = new HashMap<>();
+    protected final Listener listener;
+    private WatchService watcher;
+    private long lastValidation;
+
     @SneakyThrows
-    private static Stream<Path> walkThrough(Path path) {
-        if(Files.notExists(path)){
-            log.error("{} not found. Creating directory for it..", path);
-            Files.createDirectories(path);
+    public FileWatcherThread(
+            Map<String, Path> watchSubject,
+            Listener listener
+    ) {
+        for (var entry : watchSubject.entrySet()) {
+            var root = entry.getValue();
+            if (Files.notExists(root)) {
+                log.error("{} not found. Creating directory for it..", root);
+                Files.createDirectories(root);
+            }
+            subjectRoots.put(entry.getKey(), root.toAbsolutePath());
         }
-        return Files.walk(path);
+        this.listener = listener;
+        setName("JustBackup/FileWatcher");
+        setDaemon(true);
     }
 
     @Override
     @SneakyThrows
     public void run() {
-        try (var watcher = FileSystems.getDefault().newWatchService()) {
-            var keyToParent = new HashMap<WatchKey, Path>();
-            for (Path value : watchSubject.keySet()) {
-                var key = value.register(
-                        watcher,
-                        StandardWatchEventKinds.ENTRY_CREATE,
-                        StandardWatchEventKinds.ENTRY_MODIFY
-                );
-                keyToParent.put(key, value);
+        try (var watchService = FileSystems.getDefault().newWatchService()) {
+            this.watcher = watchService;
+            for (var entry : subjectRoots.entrySet()) {
+                registerSubject(entry.getKey(), false);
             }
+            log.info("Watching {} directories for {} subjects", keyToDir.size(), subjectRoots.size());
+            lastValidation = System.currentTimeMillis();
             for (; ; ) {
                 var key = watcher.poll(10, TimeUnit.SECONDS);
-                if (key == null) continue;
                 if (Thread.interrupted()) return;
+                if (System.currentTimeMillis() - lastValidation >= VALIDATE_INTERVAL_MILLIS) {
+                    validateRoots();
+                }
+                if (key == null) continue;
+                var dir = keyToDir.get(key);
+                var subject = dir == null ? null : watchedDirs.get(dir);
                 for (var ev : key.pollEvents()) {
-                    var kind = ev.kind();
-                    if (kind == StandardWatchEventKinds.OVERFLOW)
-                        continue;
-                    var parent = keyToParent.get(key);
-                    var subject = watchSubject.get(parent);
-                    if(subject == null) {
-                        log.warn("Ignoring changes in "+parent);
+                    if (subject == null) {
+                        log.warn("Ignoring changes in unknown watch key {}", dir);
                         continue;
                     }
-                    var path = parent.resolve(((WatchEvent<Path>) ev).context());
+                    var kind = ev.kind();
+                    if (kind == StandardWatchEventKinds.OVERFLOW) {
+                        log.warn("Watch events for {} overflowed; forcing a full backup on next run.", subject);
+                        listener.onOverflow(subject);
+                        continue;
+                    }
+                    var path = dir.resolve((Path) ev.context());
+                    log.debug("event {} on {} ({})", kind, path, subject);
                     if (Files.isRegularFile(path)) {
-                        eventListener.accept(subject, parent);
+                        listener.onChange(subject, path);
                     } else if (Files.isDirectory(path) && kind == StandardWatchEventKinds.ENTRY_CREATE) {
-                        var nK = path.register(watcher, StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_MODIFY);
-                        watchSubject.put(path, subject);
-                        keyToParent.put(nK, path);
+                        // report contained files too: they may have been created before we
+                        // managed to register and would never produce their own events
+                        registerTree(path, subject, true);
                     }
                 }
-                key.reset();
+                if (!key.reset()) {
+                    log.debug("Watch key for {} is no longer valid, dropping it", dir);
+                    keyToDir.remove(key);
+                    if (dir != null) watchedDirs.remove(dir);
+                }
             }
-        }catch (InterruptedException interruptedException){
+        } catch (InterruptedException interruptedException) {
             log.info("FileWatcherThread is quitting!");
         }
     }
+
+    /**
+     * Watch keys silently go stale when a watched root is replaced rather than modified —
+     * e.g. a world format conversion moves the directory aside and recreates it, leaving us
+     * watching an orphaned inode. Compare the root's fileKey (inode) against what we
+     * registered; on mismatch, re-register the whole subject and force a full backup since
+     * events were lost in between.
+     */
+    private void validateRoots() {
+        lastValidation = System.currentTimeMillis();
+        for (var entry : subjectRoots.entrySet()) {
+            var subject = entry.getKey();
+            var root = entry.getValue();
+            Object currentKey = fileKeyOf(root);
+            if (java.util.Objects.equals(currentKey, subjectRootKeys.get(subject))) continue;
+            log.warn("Root of subject {} ({}) was replaced or recreated; re-registering watches and forcing a full backup.",
+                    subject, root);
+            registerSubject(subject, true);
+        }
+    }
+
+    private static Object fileKeyOf(Path path) {
+        try {
+            return Files.readAttributes(path, BasicFileAttributes.class).fileKey();
+        } catch (IOException e) {
+            return null;
+        }
+    }
+
+    /**
+     * (Re-)registers a subject's whole directory tree. Existing watches of the subject are
+     * cancelled first; when {@code lostEvents} is set, the subject is flagged for a full
+     * backup because changes may have happened while we were not watching.
+     */
+    private void registerSubject(String subject, boolean lostEvents) {
+        keyToDir.entrySet().removeIf(e -> {
+            if (subject.equals(watchedDirs.get(e.getValue()))) {
+                e.getKey().cancel();
+                return true;
+            }
+            return false;
+        });
+        watchedDirs.values().removeIf(subject::equals);
+
+        var root = subjectRoots.get(subject);
+        subjectRootKeys.put(subject, fileKeyOf(root));
+        if (Files.isDirectory(root)) {
+            // no need to report existing files: the startup full backup (initial call) or the
+            // forced full backup (lostEvents) already covers them
+            registerTree(root, subject, false);
+        } else {
+            log.error("Root of subject {} ({}) does not exist, nothing is watched for it.", subject, root);
+        }
+        if (lostEvents) listener.onOverflow(subject);
+    }
+
+    private void registerTree(Path root, String subject, boolean reportFiles) {
+        try (Stream<Path> walk = Files.walk(root)) {
+            for (var p : walk.toList()) {
+                if (Files.isDirectory(p)) {
+                    var abs = p.toAbsolutePath();
+                    var key = abs.register(watcher,
+                            StandardWatchEventKinds.ENTRY_CREATE,
+                            StandardWatchEventKinds.ENTRY_MODIFY);
+                    watchedDirs.put(abs, subject);
+                    keyToDir.put(key, abs);
+                } else if (reportFiles && Files.isRegularFile(p)) {
+                    listener.onChange(subject, p);
+                }
+            }
+        } catch (IOException e) {
+            log.warn("Cannot watch directory tree {}; forcing a full backup on next run.", root, e);
+            listener.onOverflow(subject);
+        }
+    }
 }

src/main/java/io/ib67/sfcraft/FsUtil.java

@@ -0,0 +1,31 @@
+package io.ib67.sfcraft;
+
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+
+public final class FsUtil {
+    private FsUtil() {
+    }
+
+    public static void deleteRecursively(Path root) throws IOException {
+        if (Files.notExists(root)) return;
+        Files.walkFileTree(root, new SimpleFileVisitor<>() {
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                Files.deleteIfExists(file);
+                return FileVisitResult.CONTINUE;
+            }
+
+            @Override
+            public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+                if (exc != null) throw exc;
+                Files.deleteIfExists(dir);
+                return FileVisitResult.CONTINUE;
+            }
+        });
+    }
+}

src/main/java/io/ib67/sfcraft/Globals.java

@@ -6,18 +6,26 @@ import io.ib67.sfcraft.config.StorageOption;
 import io.ib67.sfcraft.config.serializer.StorageOptionSerializer;
 
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.StampedLock;
 
 public class Globals {
     /**
-     * for api users: DO NOT MUTATE THIS STATE IN WHATEVER CASE SINCE THIS IS CAUTIOUSLY ALIGNED WITH THE CURRENT THREAD MODEL
-     * AND DOING SO WILL BREAK YOUR WORLD
-     * This state indicates:
-     * 1. A minecraft auto-save is in-progress.
-     * 2. The backup is in-progress
-     * 3. Some chunks are writing to disk
+     * Guards on-disk world files during backups.
+     * <p>
+     * Chunk IO ({@code RegionFileStorage.write/flush}, running on the per-dimension IOWorker
+     * threads) holds the read lock; the backup worker holds the write lock while it snapshots
+     * files. The main thread never blocks on this lock: {@code autoSave} and manual
+     * {@code saveEverything} calls are skipped while the write lock is held (see
+     * {@code MixinMinecraftServer}), which is what makes it safe for the backup thread to
+     * wait on main-thread tasks while holding the write lock.
+     * <p>
+     * A {@link StampedLock} is used instead of a ReentrantReadWriteLock because the write
+     * stamp may be released from a different code path than the one that acquired it
+     * (early release once files are cloned to staging). None of the guarded sections are
+     * reentrant.
      */
-    public static final AtomicReference<IOState> BACKUP_LOCK = new AtomicReference<>(IOState.IDLE);
+    public static final StampedLock WORLD_IO_LOCK = new StampedLock();
+
     public static final Gson GSON = new GsonBuilder().setPrettyPrinting()
             .registerTypeAdapter(StorageOption.class, new StorageOptionSerializer(
                     Map.of("s3", StorageOption.S3.class,

src/main/java/io/ib67/sfcraft/IOState.java

@@ -1,8 +0,0 @@
-package io.ib67.sfcraft;
-
-public enum IOState {
-    SAVING_WORLD,
-    BACKUP,
-    STORAGE_SYNC,
-    IDLE
-}

src/main/java/io/ib67/sfcraft/JustBackupMod.java

@@ -4,6 +4,7 @@ import com.github.luben.zstd.Zstd;
 import com.google.gson.reflect.TypeToken;
 import io.ib67.sfcraft.config.JustBackupConfig;
 import io.ib67.sfcraft.config.StorageOption;
+import io.ib67.sfcraft.strategy.BackupChains;
 import io.ib67.sfcraft.strategy.BackupStrategy;
 import io.ib67.sfcraft.strategy.BackupTracker;
 import io.ib67.sfcraft.strategy.LocalBackupStrategy;
@@ -25,14 +26,16 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
-public class JustBackupMod implements ModInitializer {
+public class JustBackupMod implements ModInitializer, FileWatcherThread.Listener {
     public static final String MOD_ID = "justbackup";
     public static final Logger LOGGER = LoggerFactory.getLogger(MOD_ID);
     protected final Map<String, BackupSubject> backupSubjects = new HashMap<>();
@@ -44,6 +47,8 @@ public class JustBackupMod implements ModInitializer {
     protected JustBackupConfig config;
     protected BackupTracker tracker;
     protected Path backupIndexPath;
+    protected Path tmpDir;
+    protected List<Path> excludedRoots;
     protected FileWatcherThread watcherThread;
 
     @Override
@@ -62,33 +67,45 @@ public class JustBackupMod implements ModInitializer {
     @SneakyThrows
     private void doInitialize() {
         LOGGER.info("Loading backup configuration");
-        Path config1 = Path.of("config");
-        backupIndexPath = config1.resolve("just_backup_index.json");
-        var configPath = config1.resolve("just_backup.json");
+        Path configDir = Path.of("config");
+        backupIndexPath = configDir.resolve("just_backup_index.json");
+        var configPath = configDir.resolve("just_backup.json");
         if (Files.notExists(configPath.getParent()))
             Files.createDirectories(configPath.getParent());
         if (Files.notExists(configPath))
             saveConfig(configPath);
-        if (Files.notExists(backupIndexPath))
-            Files.createFile(backupIndexPath);
 
         config = Globals.GSON.fromJson(Files.readString(configPath), JustBackupConfig.class);
         config.backupSubjects().forEach((k, v) -> {
             var subjectPath = Path.of(v);
             if (Files.notExists(subjectPath)) {
-                LOGGER.error("Backup _subject {} does not exist.", v);
+                LOGGER.error("Backup subject {} does not exist.", v);
                 return;
             }
-            backupSubjects.put(k, new BackupSubject(k, subjectPath, new ArrayList<>()));
+            backupSubjects.put(k, new BackupSubject(k, subjectPath));
         });
         if (config.zstdDictPath() != null && !config.zstdDictPath().isEmpty()) {
             zstdDict = Files.readAllBytes(Path.of(config.zstdDictPath()));
         }
+        tmpDir = Path.of(config.temporaryBackupDir());
+        excludedRoots = new ArrayList<>();
+        excludedRoots.add(tmpDir);
+        if (config.option() instanceof StorageOption.Local local) {
+            excludedRoots.add(Path.of(local.saveDir()));
+        }
+
         var strategy = createStrategy();
-        this.tracker = new BackupTracker(strategy, Globals.GSON.fromJson(Files.readString(backupIndexPath), new TypeToken<>() {
-        }));
+        Map<String, Backup> initialIndex = Files.exists(backupIndexPath)
+                ? Globals.GSON.fromJson(Files.readString(backupIndexPath), new TypeToken<Map<String, Backup>>() {
+        })
+                : null;
+        this.tracker = new BackupTracker(strategy, initialIndex, backupIndexPath);
         LOGGER.info("Configuration has been loaded. Using storage option: {}", config.option().type());
-        scheduledBackupExecutor = Executors.newSingleThreadScheduledExecutor();
+        scheduledBackupExecutor = Executors.newSingleThreadScheduledExecutor(r -> {
+            var t = new Thread(r, "JustBackup/Worker");
+            t.setDaemon(false);
+            return t;
+        });
         registerWatchService();
         registerHandlers();
     }
@@ -98,7 +115,7 @@ public class JustBackupMod implements ModInitializer {
                 config.backupSubjects().entrySet().stream()
                         .map(it -> Map.entry(it.getKey(), Path.of(it.getValue())))
                         .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)),
-                this::handleFileChanges
+                this
         );
         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
             if (watcherThread.getState() != Thread.State.TERMINATED) watcherThread.interrupt();
@@ -106,12 +123,16 @@ public class JustBackupMod implements ModInitializer {
         watcherThread.start();
     }
 
-    private void handleFileChanges(String _subject, Path path) {
-        synchronized (backupSubjects) {
-            var subject = backupSubjects.get(_subject);
-            if (subject != null)
-                subject.changedFiles().add(path); // check null
-        }
+    @Override
+    public void onChange(String subjectName, Path file) {
+        var subject = backupSubjects.get(subjectName);
+        if (subject != null) subject.addChanged(file);
+    }
+
+    @Override
+    public void onOverflow(String subjectName) {
+        var subject = backupSubjects.get(subjectName);
+        if (subject != null) subject.forceFullNext();
     }
 
     @SneakyThrows
@@ -130,96 +151,168 @@ public class JustBackupMod implements ModInitializer {
                         }
                         lastBackupServerTicks = server.getTickCount();
                         if (!suspend) backupAll(config.incremental()).thenAccept(it -> {
-                            LOGGER.info("Backup success!");
-                            it.forEach((k, v) -> LOGGER.info(k + ": " + v));
+                            LOGGER.info("Scheduled backup finished.");
+                            it.forEach((k, v) -> LOGGER.info("{}: {}", k, v));
                         });
                     },
                     config.backupIntervalMinutes(), config.backupIntervalMinutes(), TimeUnit.MINUTES);
         });
-        ServerLifecycleEvents.SERVER_STOPPED.register(server -> {
-            try {
-                Files.deleteIfExists(Path.of(config.temporaryBackupDir()));
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        });
         var command = new BackupCommands(this);
         CommandRegistrationCallback.EVENT.register(command::registerCommand);
-        ServerLifecycleEvents.SERVER_STOPPED.register(server -> {
-            LOGGER.info("Shutting down the executor");
+        // Runs at the head of stopServer, before the final world save: in-flight backups must
+        // release the world IO lock before that save, or the flush would park forever.
+        ServerLifecycleEvents.SERVER_STOPPING.register(server -> {
+            suspend = true;
             scheduledBackupExecutor.shutdown();
-            if (command.restoreIssued != null) {
-                command.restoreIssued.run();
+            try {
+                if (!scheduledBackupExecutor.awaitTermination(15, TimeUnit.SECONDS)) {
+                    LOGGER.info("Waiting up to 10 minutes for the in-flight backup to finish...");
+                    if (!scheduledBackupExecutor.awaitTermination(10, TimeUnit.MINUTES)) {
+                        LOGGER.error("In-flight backup did not finish in time; its bundle may be incomplete.");
+                    }
+                }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
             }
-            tracker.close();
         });
+        ServerLifecycleEvents.SERVER_STOPPED.register(server -> {
+            if (command.restoreIssued != null) {
+                try {
+                    command.restoreIssued.run();
+                } catch (Exception e) {
+                    LOGGER.error("Backup restoration failed", e);
+                }
+            }
+            try {
+                tracker.close();
+            } catch (Exception e) {
+                LOGGER.error("Error while closing the backup storage", e);
+            }
+            cleanTemporaryDir();
+            if (watcherThread != null && watcherThread.getState() != Thread.State.TERMINATED) {
+                watcherThread.interrupt();
+            }
+        });
+    }
+
+    /**
+     * Removes only artifacts we created (staging clones, leftover bundles, temp downloads).
+     * Never deletes the directory itself: a misconfigured temporaryBackupDir pointing at the
+     * server root must not wipe the server.
+     */
+    private void cleanTemporaryDir() {
+        try {
+            FsUtil.deleteRecursively(tmpDir.resolve("staging"));
+        } catch (IOException e) {
+            LOGGER.warn("Cannot clean staging dir", e);
+        }
+        if (Files.notExists(tmpDir)) return;
+        try (Stream<Path> files = Files.list(tmpDir)) {
+            files.filter(Files::isRegularFile)
+                    .filter(p -> {
+                        var n = p.getFileName().toString();
+                        return (n.startsWith("backup_") && n.endsWith(".swb.zst")) || n.startsWith("s3_");
+                    })
+                    .forEach(p -> {
+                        try {
+                            Files.deleteIfExists(p);
+                        } catch (IOException e) {
+                            LOGGER.warn("Cannot delete leftover {}", p, e);
+                        }
+                    });
+        } catch (IOException e) {
+            LOGGER.warn("Cannot clean temporary backup dir", e);
+        }
+    }
+
+    /**
+     * Logs the message and, when enabled and a server is up, broadcasts it to players from
+     * the main thread (PlayerList is not safe to touch from the backup executor).
+     */
+    public void broadcast(Component message) {
+        LOGGER.info(message.getString());
+        if (!config.broadcast()) return;
+        var s = server;
+        if (s == null || !s.isRunning()) return;
+        s.submit(() -> s.getPlayerList().broadcastSystemMessage(message, false));
     }
 
     public CompletableFuture<Map<String, Backup>> backupAll(boolean incremental) {
         if (backupSubjects.isEmpty()) return CompletableFuture.completedFuture(Map.of());
         var map = new HashMap<String, CompletableFuture<Backup>>();
-        server.getPlayerList().broadcastSystemMessage(
-                Component.literal(" BACKUP >> ").withColor(Color.RED.getRGB()).withStyle(it -> it.withBold(true))
-                        .append(Component.nullToEmpty("The server is performing a backup, you may experience some lag.")),
-                false
-        );
+        broadcast(Component.literal(" BACKUP >> ").withColor(Color.RED.getRGB()).withStyle(it -> it.withBold(true))
+                .append(Component.nullToEmpty("The server is performing a backup, you may experience some lag.")));
         for (var entry : backupSubjects.entrySet()) {
             map.put(entry.getKey(), issueBackup(entry.getKey(), incremental));
         }
         return CompletableFuture.allOf(map.values().toArray(new CompletableFuture[0]))
                 .thenApply(it -> {
-                    var _map = new HashMap<String, Backup>();
+                    var results = new HashMap<String, Backup>();
                     map.forEach((k, v) -> {
                         var result = v.join();
                         LOGGER.info("Backup information of {}: {}", k, result);
-                        _map.put(k, result);
+                        results.put(k, result);
                     });
-                    return _map;
+                    return results;
                 });
     }
 
     public CompletableFuture<Backup> issueBackup(String subjectName, boolean incremental) {
         var subject = backupSubjects.get(subjectName);
         if (subject == null)
-            return CompletableFuture.failedFuture(new IllegalArgumentException("Unknown backup _subject: " + subjectName));
-        if (!incremental) {
-            try (var s = Files.walk(subject.root())) {
-                subject = new BackupSubject(subject.name(), subject.root(), s.filter(Files::isRegularFile)
-                        .filter(it -> !it.getFileName().toString().equals("session.lock"))
-                        .toList());
-            } catch (IOException e) {
-                return CompletableFuture.failedFuture(new IllegalArgumentException("Cannot perform full backup for " + subjectName, e));
-            }
-        }
-        var finalSubject = subject;
+            return CompletableFuture.failedFuture(new IllegalArgumentException("Unknown backup subject: " + subjectName));
         if (config.allowGunzip()) LOGGER.info("Chunk reassembler is enabled!");
-        var worker = new BackupWorker(server, tracker, Path.of(config.temporaryBackupDir()),
-                subject, c ->
+        var worker = new BackupWorker(server, tracker, tmpDir, subject, incremental, excludedRoots, c ->
                 c.allowGunzip(config.allowGunzip())
-                        .relativeRoot(finalSubject.root())
                         .allocator(ByteBufAllocator.DEFAULT)
                         .maxWorkers(config.compressionWorkerThreads())
                         .compressionLevel(config.compressionLevel())
                         .dictionary(zstdDict)
         );
-        return CompletableFuture.supplyAsync(worker, scheduledBackupExecutor).whenComplete(this::onBackupComplete);
+        return CompletableFuture.supplyAsync(() -> {
+            if (!tracker.isAvailable()) {
+                throw new IllegalStateException("Backup storage is not available (quota exceeded?), skipping backup of " + subjectName);
+            }
+            return worker.get();
+        }, scheduledBackupExecutor).whenComplete(this::onBackupComplete);
     }
 
-    @SneakyThrows
     private void onBackupComplete(@UnknownNullability Backup backup, @UnknownNullability Throwable throwable) {
         if (throwable != null) {
-            server.getPlayerList().broadcastSystemMessage(Component.literal("Backup failed. For administrators, please check your server console."), false);
-            LOGGER.error(throwable.getMessage(), throwable);
+            broadcast(Component.literal("Backup failed. For administrators, please check your server console."));
+            LOGGER.error("Backup failed", throwable);
         } else {
-            Files.writeString(backupIndexPath, Globals.GSON.toJson(tracker.getTrackedBackups()));
-            server.getPlayerList().broadcastSystemMessage(Component.literal("Backup success. " + backup), false);
+            broadcast(Component.literal("Backup success. ").append(backup.toText()));
+            rotate(backup.subject());
+        }
+    }
+
+    /**
+     * Deletes whole generations (a full backup plus its incrementals) beyond the newest
+     * {@code keepBackups} ones. Entries without chain metadata are never rotated away.
+     */
+    private void rotate(String subjectName) {
+        if (subjectName == null) return;
+        try {
+            var victims = BackupChains.rotationVictims(
+                    tracker.getTrackedBackups().values(), subjectName, config.keepBackups());
+            for (var victim : victims) {
+                LOGGER.info("Rotating out old backup {} ({})", victim.backupKey(), victim.incremental() ? "INCR" : "FULL");
+                try {
+                    tracker.deleteBackup(victim);
+                } catch (Exception e) {
+                    LOGGER.warn("Cannot delete old backup {}", victim.backupKey(), e);
+                }
+            }
+        } catch (Exception e) {
+            LOGGER.warn("Backup rotation failed for {}", subjectName, e);
         }
     }
 
     private BackupStrategy createStrategy() {
         return switch (config.option()) {
             case StorageOption.Local local -> new LocalBackupStrategy(local);
-            case StorageOption.S3 s3 -> new S3BackupStrategy(s3, Path.of(config.temporaryBackupDir()));
+            case StorageOption.S3 s3 -> new S3BackupStrategy(s3, tmpDir);
         };
     }
 
@@ -245,4 +338,4 @@ public class JustBackupMod implements ModInitializer {
         }
         Files.writeString(configPath, Globals.GSON.toJson(cfg));
     }
-}
+}

src/main/java/io/ib67/sfcraft/ReflinkCloner.java

@@ -0,0 +1,163 @@
+package io.ib67.sfcraft;
+
+import lombok.extern.log4j.Log4j2;
+
+import java.lang.foreign.Arena;
+import java.lang.foreign.FunctionDescriptor;
+import java.lang.foreign.Linker;
+import java.lang.foreign.MemoryLayout;
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.StructLayout;
+import java.lang.foreign.ValueLayout;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.VarHandle;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Clones files via the Linux {@code FICLONE} ioctl (reflink). On CoW filesystems
+ * (btrfs, XFS with reflink, OpenZFS >= 2.2) this is a metadata-only operation: the clone
+ * shares extents with the source, so staging a whole backup takes milliseconds and no
+ * extra disk space, letting the backup release the world IO lock before compressing.
+ * <p>
+ * Unsupported setups (non-Linux, ext4, overlayfs, staging dir on a different mount) are
+ * detected on first use and cached per subject; callers then fall back to packing while
+ * holding the world IO lock.
+ */
+@Log4j2
+public final class ReflinkCloner {
+    private static final long FICLONE = 0x40049409L;
+    private static final int O_RDONLY = 0;
+    private static final int O_WRONLY = 1;
+    private static final int O_CREAT = 0x40;
+    private static final int O_TRUNC = 0x200;
+    private static final int ENOENT = 2;
+    private static final int EXDEV = 18;
+
+    private static final MethodHandle OPEN;
+    private static final MethodHandle IOCTL;
+    private static final MethodHandle CLOSE;
+    private static final StructLayout CAPTURE_LAYOUT;
+    private static final VarHandle ERRNO;
+    private static final boolean AVAILABLE;
+    private static final Set<String> UNSUPPORTED_SUBJECTS = ConcurrentHashMap.newKeySet();
+
+    static {
+        MethodHandle open = null, ioctl = null, close = null;
+        StructLayout capture = null;
+        VarHandle errno = null;
+        boolean ok = false;
+        try {
+            if (System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("linux")) {
+                var linker = Linker.nativeLinker();
+                var lookup = linker.defaultLookup();
+                var captureErrno = Linker.Option.captureCallState("errno");
+                capture = Linker.Option.captureStateLayout();
+                errno = capture.varHandle(MemoryLayout.PathElement.groupElement("errno"));
+                open = linker.downcallHandle(lookup.findOrThrow("open"),
+                        FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.ADDRESS, ValueLayout.JAVA_INT, ValueLayout.JAVA_INT),
+                        captureErrno);
+                ioctl = linker.downcallHandle(lookup.findOrThrow("ioctl"),
+                        FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG, ValueLayout.JAVA_INT),
+                        captureErrno, Linker.Option.firstVariadicArg(2));
+                close = linker.downcallHandle(lookup.findOrThrow("close"),
+                        FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_INT));
+                ok = true;
+            } else {
+                log.info("Reflink fast path is Linux-only; backups will pack while holding the world IO lock.");
+            }
+        } catch (Throwable t) {
+            log.info("Reflink fast path unavailable ({}); backups will pack while holding the world IO lock.", t.toString());
+        }
+        OPEN = open;
+        IOCTL = ioctl;
+        CLOSE = close;
+        CAPTURE_LAYOUT = capture;
+        ERRNO = errno;
+        AVAILABLE = ok;
+    }
+
+    private ReflinkCloner() {
+    }
+
+    /**
+     * Clones {@code files} (which must live under {@code subjectRoot}) into
+     * {@code stagingRoot}, preserving their relative structure. Files that vanished since
+     * being listed are skipped.
+     *
+     * @return the staging root on success, or null if reflink is unsupported here — in that
+     * case any partial staging has been cleaned up and the subject is remembered as
+     * unsupported so future backups skip the attempt.
+     */
+    public static Path tryStage(String subjectName, Path subjectRoot, List<Path> files, Path stagingRoot) {
+        if (!AVAILABLE || UNSUPPORTED_SUBJECTS.contains(subjectName)) return null;
+        var absRoot = subjectRoot.toAbsolutePath().normalize();
+        try {
+            Files.createDirectories(stagingRoot);
+            for (Path file : files) {
+                var src = file.toAbsolutePath().normalize();
+                if (!Files.isRegularFile(src)) continue;
+                var dst = stagingRoot.resolve(absRoot.relativize(src));
+                Files.createDirectories(dst.getParent());
+                int err = clone(src, dst);
+                if (err == ENOENT) continue; // deleted between listing and cloning
+                if (err != 0) {
+                    markUnsupported(subjectName, src, err);
+                    FsUtil.deleteRecursively(stagingRoot);
+                    return null;
+                }
+            }
+            return stagingRoot;
+        } catch (Throwable t) {
+            log.warn("Reflink staging failed; falling back to packing under the world IO lock.", t);
+            UNSUPPORTED_SUBJECTS.add(subjectName);
+            try {
+                FsUtil.deleteRecursively(stagingRoot);
+            } catch (Exception cleanup) {
+                log.warn("Cannot clean up staging dir {}", stagingRoot, cleanup);
+            }
+            return null;
+        }
+    }
+
+    private static void markUnsupported(String subjectName, Path src, int err) {
+        UNSUPPORTED_SUBJECTS.add(subjectName);
+        if (err == EXDEV) {
+            log.warn("Reflink not possible for subject {}: {} and the temporary backup dir are on " +
+                    "different filesystems. Put temporaryBackupDir on the same filesystem as the " +
+                    "backup subject to enable the CoW fast path. Falling back to packing under the world IO lock.", subjectName, src);
+        } else {
+            log.info("Reflink unsupported for subject {} (errno {} on {}); " +
+                    "falling back to packing under the world IO lock.", subjectName, err, src);
+        }
+    }
+
+    private static int clone(Path src, Path dst) throws Throwable {
+        try (var arena = Arena.ofConfined()) {
+            var capture = arena.allocate(CAPTURE_LAYOUT);
+            int srcFd = (int) OPEN.invokeExact(capture, arena.allocateFrom(src.toString()), O_RDONLY, 0);
+            if (srcFd < 0) return errno(capture);
+            try {
+                int dstFd = (int) OPEN.invokeExact(capture, arena.allocateFrom(dst.toString()),
+                        O_WRONLY | O_CREAT | O_TRUNC, 0644);
+                if (dstFd < 0) return errno(capture);
+                try {
+                    int result = (int) IOCTL.invokeExact(capture, dstFd, FICLONE, srcFd);
+                    return result == 0 ? 0 : errno(capture);
+                } finally {
+                    var ignored = (int) CLOSE.invokeExact(dstFd);
+                }
+            } finally {
+                var ignored = (int) CLOSE.invokeExact(srcFd);
+            }
+        }
+    }
+
+    private static int errno(MemorySegment capture) {
+        return (int) ERRNO.get(capture, 0L);
+    }
+}

src/main/java/io/ib67/sfcraft/mixin/MixinMinecraftServer.java

@@ -1,14 +1,22 @@
 package io.ib67.sfcraft.mixin;
 
 import io.ib67.sfcraft.Globals;
-import io.ib67.sfcraft.IOState;
+import io.ib67.sfcraft.JustBackupMod;
 import net.minecraft.server.MinecraftServer;
 import org.spongepowered.asm.mixin.Mixin;
 import org.spongepowered.asm.mixin.Shadow;
 import org.spongepowered.asm.mixin.injection.At;
 import org.spongepowered.asm.mixin.injection.Inject;
 import org.spongepowered.asm.mixin.injection.callback.CallbackInfo;
+import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable;
 
+/**
+ * Skips main-thread saves while a backup holds the write lock. The main thread must never
+ * block on {@link Globals#WORLD_IO_LOCK}: a save request would join the IOWorkers, which
+ * are parked on the read lock, wedging the main thread until the backup finishes (and
+ * tripping the watchdog). The world was saved right before the backup acquired the lock,
+ * so skipping loses nothing.
+ */
 @Mixin(MinecraftServer.class)
 public abstract class MixinMinecraftServer {
 
@@ -19,23 +27,21 @@ public abstract class MixinMinecraftServer {
     protected abstract int computeNextAutosaveInterval();
 
     @Inject(method = "autoSave", at = @At("HEAD"), cancellable = true)
-    private void backup$checkSaving(CallbackInfo ci) {
-        // is already backing up
-        this.ticksUntilAutosave = computeNextAutosaveInterval();
-        IOState witness;
-        while ((witness = Globals.BACKUP_LOCK.compareAndExchange(IOState.IDLE, IOState.SAVING_WORLD)) == IOState.STORAGE_SYNC) {
-            try {
-                Thread.sleep(1000);
-            } catch (InterruptedException e) {
-                System.err.println("Error while waiting for chunk sync. " + e);
-                ci.cancel();
-            }
+    private void backup$skipAutoSaveDuringBackup(CallbackInfo ci) {
+        if (Globals.WORLD_IO_LOCK.isWriteLocked()) {
+            // vanilla resets this at the head of autoSave; replicate it on the cancel path
+            // so the autosave doesn't re-trigger every tick
+            this.ticksUntilAutosave = computeNextAutosaveInterval();
+            JustBackupMod.LOGGER.info("Backup in progress, skipping this auto-save.");
+            ci.cancel();
         }
-        if (witness == IOState.BACKUP) ci.cancel();
     }
 
-    @Inject(method = "autoSave", at = @At("RETURN"))
-    private void backup$disableProtect(CallbackInfo ci) {
-        Globals.BACKUP_LOCK.setRelease(IOState.IDLE);
+    @Inject(method = "saveEverything", at = @At("HEAD"), cancellable = true)
+    private void backup$skipManualSaveDuringBackup(boolean suppressLog, boolean flush, boolean force, CallbackInfoReturnable<Boolean> cir) {
+        if (Globals.WORLD_IO_LOCK.isWriteLocked()) {
+            JustBackupMod.LOGGER.warn("Backup in progress, skipping this save request. The world was already saved when the backup started.");
+            cir.setReturnValue(false);
+        }
     }
 }

src/main/java/io/ib67/sfcraft/mixin/MixinRegionBasedStorage.java

@@ -1,62 +1,38 @@
 package io.ib67.sfcraft.mixin;
 
+import com.llamalad7.mixinextras.injector.wrapmethod.WrapMethod;
+import com.llamalad7.mixinextras.injector.wrapoperation.Operation;
 import io.ib67.sfcraft.Globals;
-import io.ib67.sfcraft.IOState;
-import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
-import lombok.SneakyThrows;
-import net.minecraft.world.level.chunk.storage.RegionFile;
+import net.minecraft.nbt.CompoundTag;
+import net.minecraft.world.level.ChunkPos;
 import net.minecraft.world.level.chunk.storage.RegionFileStorage;
-import org.spongepowered.asm.mixin.Final;
 import org.spongepowered.asm.mixin.Mixin;
-import org.spongepowered.asm.mixin.Overwrite;
-import org.spongepowered.asm.mixin.Shadow;
 
+/**
+ * Holds the read side of {@link Globals#WORLD_IO_LOCK} around region file writes and flushes.
+ * Multiple IOWorker threads (one per dimension and storage kind) may run concurrently under
+ * the read lock; while the backup worker holds the write lock, all region IO parks here and
+ * resumes untouched afterwards.
+ */
 @Mixin(RegionFileStorage.class)
 public abstract class MixinRegionBasedStorage {
-    @Shadow
-    @Final
-    private Long2ObjectLinkedOpenHashMap<RegionFile> regionCache;
-
-    /**
-     * @author iceBear67
-     * @reason to wrap the saving process with an exception handler.
-     */
-    @Overwrite
-    @SneakyThrows
-    public void flush() {
-        // allowed states:
-        // STORAGE_SYNC, AUTO_SAVE, IDLE
-        // we try
-        var witness = Globals.BACKUP_LOCK.getAndSet(IOState.STORAGE_SYNC);
-        if (witness == IOState.BACKUP) {
-            // yield out and keep blocking.
-            // The backup worker will wait for us setting this back to backup.
-            int i =0;
-            Globals.BACKUP_LOCK.set(IOState.BACKUP);
-            while (Globals.BACKUP_LOCK.getAcquire() == IOState.BACKUP) {
-                if(i==0){
-                    System.out.println("Server is performing backup, ChunkIO will wait for it.");
-                }
-                if(i++ > 3000){
-                    System.err.println("WARNING: BACKUP_LOCK is holding for 10 minutes!");
-                    System.err.println("ChunkIO will not wait for backup anymore, writting to disk...");
-                    System.err.println("You may adjust this behaviour in configuration");
-                    break;
-                }
-                Thread.sleep(2000);
-            }
-        }
-        // program point: IDLE, AUTOSAVE, STORAGESYNC
+    @WrapMethod(method = "write")
+    private void backup$guardWrite(ChunkPos pos, CompoundTag tag, Operation<Void> original) {
+        long stamp = Globals.WORLD_IO_LOCK.readLock();
         try {
-            for (RegionFile value : this.regionCache.values()) {
-                value.flush();
-            }
+            original.call(pos, tag);
         } finally {
-            // we assert that there are no other threads calling sync()
-            // so we can just jump back to the witness value.
-            // witness: AUTO_SAVE or IDLE
-            Globals.BACKUP_LOCK.compareAndExchange(IOState.STORAGE_SYNC, witness);
-            Globals.BACKUP_LOCK.setRelease(witness);
+            Globals.WORLD_IO_LOCK.unlockRead(stamp);
+        }
+    }
+
+    @WrapMethod(method = "flush")
+    private void backup$guardFlush(Operation<Void> original) {
+        long stamp = Globals.WORLD_IO_LOCK.readLock();
+        try {
+            original.call();
+        } finally {
+            Globals.WORLD_IO_LOCK.unlockRead(stamp);
         }
     }
 }

src/main/java/io/ib67/sfcraft/strategy/BackupChains.java

@@ -0,0 +1,84 @@
+package io.ib67.sfcraft.strategy;
+
+import io.ib67.sfcraft.Backup;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Pure functions over the backup index: resolving restore chains (full base + following
+ * incrementals) and grouping backups into generations for rotation. Legacy entries
+ * (created before subject/createdAt existed) are excluded from both — they are restorable
+ * standalone but never chained or rotated away.
+ */
+public final class BackupChains {
+    private BackupChains() {
+    }
+
+    /**
+     * Resolves the list of bundles that must be extracted, in order, to restore {@code target}.
+     * A full backup restores alone; an incremental needs its most recent full base and every
+     * incremental in between.
+     *
+     * @throws IllegalStateException if the target is an incremental without a reachable base
+     */
+    public static List<Backup> resolveChain(Collection<Backup> tracked, Backup target) {
+        if (!target.incremental()) return List.of(target);
+        if (target.legacy()) {
+            throw new IllegalStateException("Backup " + target.backupKey()
+                    + " lacks chain metadata (created by an older version) and cannot be restored incrementally.");
+        }
+        var chain = new ArrayList<Backup>();
+        tracked.stream()
+                .filter(b -> !b.legacy() && Objects.equals(target.subject(), b.subject()))
+                .filter(b -> b.createdAt() < target.createdAt())
+                .sorted(Comparator.comparingLong(Backup::createdAt))
+                .forEach(b -> {
+                    if (!b.incremental()) chain.clear(); // a newer full base supersedes older history
+                    chain.add(b);
+                });
+        chain.add(target);
+        if (chain.getFirst().incremental()) {
+            throw new IllegalStateException("No full backup found before " + target.backupKey()
+                    + "; cannot restore an incremental backup without its full base.");
+        }
+        return List.copyOf(chain);
+    }
+
+    /**
+     * Groups a subject's backups into generations: each full backup starts a new generation
+     * that collects the incrementals after it. Incrementals older than the first full backup
+     * (orphans) belong to no generation.
+     */
+    public static List<List<Backup>> generations(Collection<Backup> tracked, String subject) {
+        var result = new ArrayList<List<Backup>>();
+        List<Backup> current = null;
+        var sorted = tracked.stream()
+                .filter(b -> !b.legacy() && Objects.equals(subject, b.subject()))
+                .sorted(Comparator.comparingLong(Backup::createdAt))
+                .toList();
+        for (var b : sorted) {
+            if (!b.incremental()) {
+                current = new ArrayList<>();
+                result.add(current);
+            }
+            if (current != null) current.add(b);
+        }
+        return result;
+    }
+
+    /**
+     * @return the backups that fall out of the newest {@code keepGenerations} generations and
+     * should be deleted. Whole generations only — an incremental is never separated from its base.
+     */
+    public static List<Backup> rotationVictims(Collection<Backup> tracked, String subject, int keepGenerations) {
+        var generations = generations(tracked, subject);
+        if (generations.size() <= keepGenerations) return List.of();
+        return generations.subList(0, generations.size() - keepGenerations).stream()
+                .flatMap(List::stream)
+                .toList();
+    }
+}

src/main/java/io/ib67/sfcraft/strategy/BackupStrategy.java

@@ -6,10 +6,16 @@ import java.nio.file.Path;
 
 public interface BackupStrategy {
     /**
-     * create a backup
-     * @return a key to this backup
+     * Stores the finished bundle.
+     *
+     * @param subject     name of the backup subject
+     * @param source      directory the bundle was created from
+     * @param bundle      the bundle file to store; the caller deletes it afterwards
+     * @param incremental whether the bundle only contains changed files
+     * @param createdAt   epoch millis the backup was taken at
+     * @return metadata describing the stored backup
      */
-    Backup createBackup(String subject, Path source, Path bundle);
+    Backup createBackup(String subject, Path source, Path bundle, boolean incremental, long createdAt);
 
     void recoverBackup(Backup backup, Path restorePath);
 

src/main/java/io/ib67/sfcraft/strategy/BackupTracker.java

@@ -1,28 +1,46 @@
 package io.ib67.sfcraft.strategy;
 
 import io.ib67.sfcraft.Backup;
+import io.ib67.sfcraft.Globals;
 import lombok.Getter;
-import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
 
+import java.io.IOException;
+import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.HashMap;
+import java.nio.file.StandardCopyOption;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
 
+/**
+ * Wraps a storage strategy with an in-memory index of known backups, persisted atomically
+ * to {@code indexPath} after every mutation. The map is concurrent: the backup executor
+ * writes while the main thread reads it for commands and tab completion.
+ */
+@Log4j2
 public class BackupTracker implements BackupStrategy {
     protected final BackupStrategy underlying;
     @Getter
-    protected final Map<String, Backup> trackedBackups;
+    protected final Map<String, Backup> trackedBackups = new ConcurrentHashMap<>();
+    protected final Path indexPath;
 
-    public BackupTracker(BackupStrategy underlying, Map<String, Backup> trackedBackups) {
+    public BackupTracker(BackupStrategy underlying, Map<String, Backup> initial, Path indexPath) {
         this.underlying = Objects.requireNonNull(underlying);
-        this.trackedBackups = trackedBackups == null ? new HashMap<>() : trackedBackups;
+        this.indexPath = Objects.requireNonNull(indexPath);
+        if (initial != null) {
+            initial.forEach((k, v) -> {
+                if (k != null && v != null) trackedBackups.put(k, v);
+            });
+        }
     }
 
     @Override
-    public Backup createBackup(String subject, Path from, Path pathToBundle) {
-        var result = underlying.createBackup(subject, from, pathToBundle);
+    public Backup createBackup(String subject, Path source, Path bundle, boolean incremental, long createdAt) {
+        var result = underlying.createBackup(subject, source, bundle, incremental, createdAt);
         trackedBackups.put(result.backupKey(), result);
+        persist();
         return result;
     }
 
@@ -35,6 +53,26 @@ public class BackupTracker implements BackupStrategy {
     public void deleteBackup(Backup backup) {
         underlying.deleteBackup(backup);
         trackedBackups.remove(backup.backupKey());
+        persist();
+    }
+
+    /**
+     * Writes the index to a temp file first and atomically moves it into place, so a crash
+     * mid-write can never leave a corrupt index behind (which would disable the mod on the
+     * next startup).
+     */
+    protected synchronized void persist() {
+        try {
+            var tmp = indexPath.resolveSibling(indexPath.getFileName() + ".tmp");
+            Files.writeString(tmp, Globals.GSON.toJson(trackedBackups));
+            try {
+                Files.move(tmp, indexPath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+            } catch (AtomicMoveNotSupportedException e) {
+                Files.move(tmp, indexPath, StandardCopyOption.REPLACE_EXISTING);
+            }
+        } catch (IOException e) {
+            log.error("Cannot persist backup index to {}", indexPath, e);
+        }
     }
 
     @Override

src/main/java/io/ib67/sfcraft/strategy/LocalBackupStrategy.java

@@ -25,24 +25,23 @@ public class LocalBackupStrategy implements BackupStrategy {
 
     @Override
     @SneakyThrows
-    public Backup createBackup(String subject, Path from, Path pathToBundle) {
+    public Backup createBackup(String subject, Path source, Path bundle, boolean incremental, long createdAt) {
         var parent = backupParentRoot.resolve(subject);
+        var fileName = bundle.getFileName().toString();
         var result = new Backup(
-                pathToBundle.getFileName().toString(),
-                Path.of(subject).resolve(pathToBundle.getFileName()).normalize().toString(),
-                "local", from.toString(), false, Files.size(pathToBundle));
-        if(Files.notExists(parent)) {
+                fileName,
+                Path.of(subject).resolve(fileName).normalize().toString(),
+                "local", source.toString(), subject, incremental, createdAt, Files.size(bundle));
+        if (Files.notExists(parent)) {
             Files.createDirectories(parent);
         }
-        //todo new name
-        Files.move(pathToBundle, parent.resolve(pathToBundle.getFileName()));
+        Files.move(bundle, parent.resolve(fileName));
         return result;
     }
 
     @Override
     @SneakyThrows
     public void recoverBackup(Backup backup, Path restorePath) {
-        // check if it is compressed
         var backupFile = backupParentRoot.resolve(backup.backupKey());
         if (Files.notExists(backupFile)) throw new IllegalArgumentException("Backup " + backupFile + " does not exist");
         BundleReader.builder().build().extract(backupFile, restorePath);
@@ -51,7 +50,7 @@ public class LocalBackupStrategy implements BackupStrategy {
     @SneakyThrows
     @Override
     public void deleteBackup(Backup backup) {
-        Validate.isTrue("local".equals(backup.backupKey()));
+        Validate.isTrue("local".equals(backup.type()), "Not a local backup: %s", backup.backupKey());
         var bundlePath = backupParentRoot.resolve(backup.backupKey());
         Files.deleteIfExists(bundlePath);
     }
@@ -61,14 +60,14 @@ public class LocalBackupStrategy implements BackupStrategy {
     public boolean isAvailable() {
         if (option.diskSizeQuotaBytes() == 0) return true;
         try (var f = Files.walk(backupParentRoot)) {
-            if (f.mapToLong(it->{
+            if (f.filter(Files::isRegularFile).mapToLong(it -> {
                 try {
                     return Files.size(it);
                 } catch (IOException e) {
                     throw new RuntimeException(e);
                 }
             }).sum() >= option.diskSizeQuotaBytes()) {
-                log.warn("Have no space to set another backup! ");
+                log.warn("Backup disk quota ({} bytes) exceeded, refusing to create new backups!", option.diskSizeQuotaBytes());
                 return false;
             }
         }

src/main/java/io/ib67/sfcraft/strategy/S3BackupStrategy.java

@@ -7,20 +7,18 @@ import lombok.SneakyThrows;
 import lombok.extern.log4j.Log4j2;
 import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
 import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.exception.SdkException;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3Client;
 
-import java.io.IOException;
 import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.util.Objects;
-import java.util.concurrent.atomic.AtomicBoolean;
 
 @Log4j2
 public class S3BackupStrategy implements BackupStrategy {
-    protected final AtomicBoolean uploading = new AtomicBoolean(false);
     protected final StorageOption.S3 option;
     protected final S3Client s3;
     protected final Path temporaryDownloadPath;
@@ -45,45 +43,42 @@ public class S3BackupStrategy implements BackupStrategy {
 
     @Override
     @SneakyThrows
-    public Backup createBackup(String subject, Path from, Path pathToBundle) {
-        if (!uploading.compareAndSet(false, true)) {
-            throw new IllegalStateException("Last upload is not done yet! keep waiting...");
-        }
-        int i = 0;
-        do {
+    public Backup createBackup(String subject, Path source, Path bundle, boolean incremental, long createdAt) {
+        // bundle names embed the creation timestamp, so keys are unique and retries idempotent
+        var object = option.prefix() + "/" + bundle.getFileName();
+        int attempt = 0;
+        while (true) {
             try {
-                var object = option.prefix() + "/" + pathToBundle.getFileName().toString();
                 s3.putObject(o -> o.bucket(option.bucket())
                         .contentType("application/octet-stream")
-                        .ifNoneMatch("*") // to avoid content overriding
-                        .key(object).build(), pathToBundle);
-                return new Backup(pathToBundle.getFileName().toString(),
+                        .key(object), bundle);
+                return new Backup(bundle.getFileName().toString(),
                         object, "s3",
-                        from.toString(),
-                        false,
-                        Files.size(pathToBundle));
-            } catch (IOException e) {
-                if (i >= option.retryAmount()) {
-                    throw new IllegalStateException("Cannot upload " + pathToBundle + " to S3");
+                        source.toString(),
+                        subject, incremental, createdAt,
+                        Files.size(bundle));
+            } catch (SdkException e) {
+                if (attempt >= option.retryAmount()) {
+                    throw new IllegalStateException("Cannot upload " + bundle + " to S3 after "
+                            + attempt + " retries", e);
                 }
-                var nextTry = Math.pow(2, i++);
-                log.error("Error uploading S3. trying in {}s later. ({}/{})", nextTry, i, option.retryAmount(), e);
-                Thread.sleep((long) (nextTry * 1000));
-            } catch (Exception e) {
-                throw new IllegalStateException("Unrecoverable exception occurred when uploading " + pathToBundle, e);
-            } finally {
-                uploading.set(false);
+                var backoffSeconds = (long) Math.pow(2, attempt++);
+                log.error("Error uploading to S3, retrying in {}s ({}/{})",
+                        backoffSeconds, attempt, option.retryAmount(), e);
+                Thread.sleep(backoffSeconds * 1000);
             }
-        } while (true);
+        }
     }
 
     @Override
     @SneakyThrows
     public void recoverBackup(Backup backup, Path restorePath) {
-        var resp = s3.getObject(b -> b.bucket(option.bucket()).key(backup.backupKey()));
         var target = temporaryDownloadPath.resolve("s3_" + System.currentTimeMillis());
-        try (var fs = Files.newOutputStream(target, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
-            resp.transferTo(fs);
+        try {
+            try (var resp = s3.getObject(b -> b.bucket(option.bucket()).key(backup.backupKey()));
+                 var fs = Files.newOutputStream(target, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
+                resp.transferTo(fs);
+            }
             BundleReader.builder().build().extract(target, restorePath);
         } finally {
             Files.deleteIfExists(target);
@@ -98,10 +93,6 @@ public class S3BackupStrategy implements BackupStrategy {
 
     @Override
     public boolean isAvailable() {
-        if (uploading.get()) {
-            log.warn("Last S3 Backup is still uploading!");
-            return false;
-        }
         return true;
     }
 

src/test/java/io/ib67/sfcraft/TestBackupChains.java

@@ -0,0 +1,88 @@
+package io.ib67.sfcraft;
+
+import io.ib67.sfcraft.strategy.BackupChains;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class TestBackupChains {
+    private static Backup full(String key, String subject, long at) {
+        return new Backup(key, key, "local", "world", subject, false, at, 1);
+    }
+
+    private static Backup incr(String key, String subject, long at) {
+        return new Backup(key, key, "local", "world", subject, true, at, 1);
+    }
+
+    private static Backup legacy(String key) {
+        return new Backup(key, key, "local", "world", null, false, 0, 1);
+    }
+
+    @Test
+    public void fullBackupRestoresAlone() {
+        var full = full("f1", "world", 100);
+        var all = List.of(full, incr("i1", "world", 200));
+        assertEquals(List.of(full), BackupChains.resolveChain(all, full));
+    }
+
+    @Test
+    public void incrementalChainsBackToItsBase() {
+        var f1 = full("f1", "world", 100);
+        var i1a = incr("i1a", "world", 200);
+        var i1b = incr("i1b", "world", 300);
+        var f2 = full("f2", "world", 400);
+        var i2a = incr("i2a", "world", 500);
+        var all = List.of(i2a, f1, i1b, f2, i1a); // deliberately unordered
+
+        assertEquals(List.of(f1, i1a, i1b), BackupChains.resolveChain(all, i1b));
+        assertEquals(List.of(f2, i2a), BackupChains.resolveChain(all, i2a));
+    }
+
+    @Test
+    public void chainIgnoresOtherSubjects() {
+        var f1 = full("f1", "world", 100);
+        var otherFull = full("fo", "nether", 150);
+        var i1 = incr("i1", "world", 200);
+        assertEquals(List.of(f1, i1), BackupChains.resolveChain(List.of(f1, otherFull, i1), i1));
+    }
+
+    @Test
+    public void orphanIncrementalIsRejected() {
+        var i1 = incr("i1", "world", 200);
+        assertThrows(IllegalStateException.class,
+                () -> BackupChains.resolveChain(List.of(i1), i1));
+    }
+
+    @Test
+    public void legacyEntryRestoresStandalone() {
+        var old = legacy("old");
+        assertEquals(List.of(old), BackupChains.resolveChain(List.of(old), old));
+    }
+
+    @Test
+    public void rotationKeepsWholeGenerations() {
+        var f1 = full("f1", "world", 100);
+        var i1 = incr("i1", "world", 200);
+        var f2 = full("f2", "world", 300);
+        var i2 = incr("i2", "world", 400);
+        var f3 = full("f3", "world", 500);
+        var all = List.of(f1, i1, f2, i2, f3);
+
+        assertEquals(List.of(), BackupChains.rotationVictims(all, "world", 3));
+        assertEquals(List.of(f1, i1), BackupChains.rotationVictims(all, "world", 2));
+        assertEquals(List.of(f1, i1, f2, i2), BackupChains.rotationVictims(all, "world", 1));
+    }
+
+    @Test
+    public void rotationNeverTouchesLegacyOrOrphans() {
+        var orphan = incr("orphan", "world", 50);
+        var old = legacy("old");
+        var f1 = full("f1", "world", 100);
+        var f2 = full("f2", "world", 200);
+        var all = List.of(orphan, old, f1, f2);
+
+        assertEquals(List.of(f1), BackupChains.rotationVictims(all, "world", 1));
+    }
+}