diff --git a/build.gradle b/build.gradle index 646139b..de4b57b 100644 --- a/build.gradle +++ b/build.gradle @@ -71,6 +71,13 @@ dependencies { // https://mvnrepository.com/artifact/software.amazon.awssdk/s3 shadow project(":bundler") shadow 'software.amazon.awssdk:s3:2.34.0' + + testImplementation 'org.junit.jupiter:junit-jupiter:5.13.4' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' +} + +test { + useJUnitPlatform() } configurations.implementation.extendsFrom(configurations.shadow) shadowJar { diff --git a/bundler/src/main/java/io/ib67/sfcraft/bundler/BundleReader.java b/bundler/src/main/java/io/ib67/sfcraft/bundler/BundleReader.java index 0e618d1..cb47f1c 100644 --- a/bundler/src/main/java/io/ib67/sfcraft/bundler/BundleReader.java +++ b/bundler/src/main/java/io/ib67/sfcraft/bundler/BundleReader.java @@ -6,7 +6,9 @@ import io.ib67.sfcraft.bundler.region.RegionParser; import io.netty.buffer.ByteBufAllocator; import lombok.Builder; +import java.io.EOFException; import java.io.IOException; +import java.io.InputStream; import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; @@ -32,8 +34,14 @@ public class BundleReader { this.dict = dict; } + /** + * Extracts a bundle into {@code destination}. Existing files are overwritten, which is + * what makes chained restores work: apply the full base first, then each incremental in + * order — later bundles win. + */ public void extract(Path bundle, Path destination) throws IOException { var readMaps = new HashMap(); + var destRoot = destination.toAbsolutePath().normalize(); try (var zstd = new ZstdInputStreamNoFinalizer(Files.newInputStream(bundle, StandardOpenOption.READ)); var in = new EntryInputStream(zstd)) { if (dict != null) zstd.setDict(dict); @@ -42,62 +50,101 @@ public class BundleReader { var time = entry.time(); if (time < minTime || time > maxTime) { System.err.println("Skipping entry " + entry.name() + " (too old or too new)"); + skipExactly(in, entry.length()); continue; } var lastSeen = readMaps.get(entry.name()); if (lastSeen != null && skipOldEntry && entry.time() < lastSeen) { - System.err.println("Skipping entry " + entry.name() + " (older than )"); + System.err.println("Skipping entry " + entry.name() + " (older than seen)"); + skipExactly(in, entry.length()); continue; } readMaps.put(entry.name(), time); - var path = destination.resolve(entry.name()); // todo prevent path underflow + var path = resolveSafely(destRoot, entry.name()); var parentDir = path.getParent(); if (Files.notExists(parentDir)) Files.createDirectories(parentDir); if ((entry.attribute() & BundleEntry.ATTR_SYMLINK) != 0) { - var toRead = (int) entry.length(); - var strArr = new byte[toRead]; - while ((toRead -= in.read(strArr, (int) (entry.length() - toRead), toRead)) > 0) ; - var target = new String(strArr); - var targetPath = Path.of(target); - Files.createSymbolicLink(targetPath, path); + var strArr = new byte[(int) entry.length()]; + readExactly(in, strArr); + var targetPath = Path.of(new String(strArr)); + Files.deleteIfExists(path); + Files.createSymbolicLink(path, targetPath); } else if ((entry.attribute() & BundleEntry.ATTR_REASSEMBLE) != 0) { var entryData = allocator.buffer(); - var toRead = entry.length(); - var buffer = this.buffer; - while (toRead > 0) { - var read = in.read(buffer, 0, (int) Math.min(buffer.length, toRead)); - toRead -= read; - entryData.writeBytes(buffer, 0, read); - } - try (var reassembler = new RegionParser(entryData, allocator)) { - var result = reassembler.writeReassembled(RegionFile.CompressType.ZLIB); - try (var fc = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { - result.readBytes(fc, 0, result.readableBytes()); - } finally { - result.release(); + try { + var toRead = entry.length(); + var buffer = this.buffer; + while (toRead > 0) { + var read = in.read(buffer, 0, (int) Math.min(buffer.length, toRead)); + if (read < 0) throw new EOFException("Truncated bundle while reading " + entry.name()); + toRead -= read; + entryData.writeBytes(buffer, 0, read); } - } catch (Exception e) { - try (var ch = FileChannel.open(Path.of("dump.mca"), StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { - entryData.readerIndex(0); - entryData.readBytes(ch, 0, entryData.readableBytes()); + try (var reassembler = new RegionParser(entryData, allocator)) { + var result = reassembler.writeReassembled(RegionFile.CompressType.ZLIB); + try (var fc = FileChannel.open(path, StandardOpenOption.CREATE, + StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { + result.readBytes(fc, 0, result.readableBytes()); + } finally { + result.release(); + } + } catch (Exception e) { + var dump = Files.createTempFile("justbackup_dump_", ".mca"); + try (var ch = FileChannel.open(dump, StandardOpenOption.WRITE)) { + entryData.readerIndex(0); + entryData.readBytes(ch, 0, entryData.readableBytes()); + } + throw new IOException("Error while reassembling " + entry.name() + ", dumped to " + dump, e); } - throw new IOException("Error while reassembling " + entry.name(), e); + } finally { + entryData.release(); } } else { writePlain(path, entry, in); } } + } + } + /** + * Resolves an entry name inside the destination, refusing names that escape it + * (absolute paths or {@code ..} traversal). + */ + private static Path resolveSafely(Path destRoot, String entryName) throws IOException { + var resolved = destRoot.resolve(entryName).normalize(); + if (!resolved.startsWith(destRoot)) { + throw new IOException("Entry " + entryName + " escapes the destination directory, refusing to extract"); + } + return resolved; + } + + private static void readExactly(InputStream in, byte[] target) throws IOException { + int off = 0; + while (off < target.length) { + int read = in.read(target, off, target.length - off); + if (read < 0) throw new EOFException("Truncated bundle"); + off += read; + } + } + + private void skipExactly(InputStream in, long length) throws IOException { + var buffer = this.buffer; + while (length > 0) { + int read = in.read(buffer, 0, (int) Math.min(buffer.length, length)); + if (read < 0) throw new EOFException("Truncated bundle"); + length -= read; } } private void writePlain(Path path, BundleEntry entry, EntryInputStream in) throws IOException { var buffer = this.buffer; - try (var fc = Files.newOutputStream(path, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { + try (var fc = Files.newOutputStream(path, StandardOpenOption.CREATE, + StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { var toRead = entry.length(); while (toRead > 0) { var read = in.read(buffer, 0, (int) Math.min(buffer.length, toRead)); + if (read < 0) throw new EOFException("Truncated bundle while reading " + entry.name()); toRead -= read; fc.write(buffer, 0, read); } diff --git a/bundler/src/main/java/io/ib67/sfcraft/bundler/BundleWriter.java b/bundler/src/main/java/io/ib67/sfcraft/bundler/BundleWriter.java index ee50050..a6de9d3 100644 --- a/bundler/src/main/java/io/ib67/sfcraft/bundler/BundleWriter.java +++ b/bundler/src/main/java/io/ib67/sfcraft/bundler/BundleWriter.java @@ -12,9 +12,12 @@ import lombok.Builder; import lombok.SneakyThrows; import java.io.*; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.*; import java.util.concurrent.*; import java.util.function.UnaryOperator; @@ -131,14 +134,40 @@ public class BundleWriter implements Closeable { Iterable path, UnaryOperator config ) throws IOException { - try (var writer = config.apply(BundleWriter.builder()) - .outputStream(Files.newOutputStream(pathToBundle)) - .build()) { - for (var p : path) { - if (!Files.isRegularFile(p) || Files.size(p) == 0) continue; - var s = p.toString(); - writer.write(p, (s.contains("region") && s.endsWith(".mca")) || s.endsWith(".dat")); + try (var channel = FileChannel.open(pathToBundle, + StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { + try (var writer = config.apply(BundleWriter.builder()) + .outputStream(new NonClosingOutputStream(Channels.newOutputStream(channel))) + .build()) { + for (var p : path) { + if (!Files.isRegularFile(p) || Files.size(p) == 0) continue; + var s = p.toString(); + writer.write(p, (s.contains("region") && s.endsWith(".mca")) || s.endsWith(".dat")); + } } + // the zstd epilogue is flushed by writer.close(); fsync before closing so a crash + // right after "backup success" cannot leave a hollow bundle behind + channel.force(true); + } + } + + /** + * Lets the zstd stream be closed (writing its epilogue) without closing the underlying + * channel, so the caller can fsync it afterwards. + */ + private static final class NonClosingOutputStream extends FilterOutputStream { + NonClosingOutputStream(OutputStream out) { + super(out); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + } + + @Override + public void close() throws IOException { + out.flush(); } } @@ -191,8 +220,10 @@ public class BundleWriter implements Closeable { result.readBytes(out, length); result.release(); // all done - } catch (IOException e) { - System.err.println("Failed to parse region " + path + ": " + e); + } catch (Exception e) { + // any parse failure (not only IOException) must fall back to a plain copy instead + // of failing the whole backup + System.err.println("Failed to parse region " + path + ", storing it as-is: " + e); finalResult.release(); writePlain(path); return; @@ -205,7 +236,8 @@ public class BundleWriter implements Closeable { private void writePlain(Path path) throws IOException { var rPath = relativeRoot.toAbsolutePath().relativize(path.toAbsolutePath()); var outputStream = this.outputStream; - outputStream.writeEntryHeader(rPath.toString(), (short) 0, Files.size(path), 0); + outputStream.writeEntryHeader(rPath.toString(), (short) 0, Files.size(path), + Files.getLastModifiedTime(path).toMillis()); var read = 0; try (var fs = Files.newInputStream(path)) { var buffer = BundleWriter.buffer.get(); diff --git a/bundler/src/main/java/io/ib67/sfcraft/bundler/region/RegionFile.java b/bundler/src/main/java/io/ib67/sfcraft/bundler/region/RegionFile.java index 378b91c..88362d3 100644 --- a/bundler/src/main/java/io/ib67/sfcraft/bundler/region/RegionFile.java +++ b/bundler/src/main/java/io/ib67/sfcraft/bundler/region/RegionFile.java @@ -45,31 +45,31 @@ public class RegionFile implements Closeable { return (short) ((z & 0b111111) << 6 | (x & 0b111111)); } - // this retains the input bytebuf. public void writeChunk(int chunkX, int chunkZ, CompressType type, ByteBuf data) throws IOException { - if (chunkX > 32 || chunkZ > 32 || chunkX < 0 || chunkZ < 0) + if (chunkX >= 32 || chunkZ >= 32 || chunkX < 0 || chunkZ < 0) throw new IllegalArgumentException("Invalid chunkX+chunkZ: " + chunkX + " " + chunkZ); var key = getChunkKey(chunkX, chunkZ); - var buf = allocator.buffer(5 + data.readableBytes()); + ByteBuf payload; if (type == CompressType.NONE) { - buf.writeInt(data.readableBytes()); - buf.writeByte(type.index); - buf.writeBytes(data); + payload = allocator.buffer(data.readableBytes()); + payload.writeBytes(data); } else { - try (var out = findCompressor(type, new ByteBufOutputStream(buf))) { - while (data.readableBytes() > 0) data.readBytes(out, data.readableBytes()); - var head = allocator.buffer(5); - head.writeInt(buf.readableBytes()); - head.writeByte(type.index); - var composite = allocator.compositeBuffer(); - composite.addComponent(true, head); - composite.addComponent(true, buf); + payload = allocator.buffer(); + try (var out = findCompressor(type, new ByteBufOutputStream(payload))) { + data.readBytes(out, data.readableBytes()); } catch (IOException e) { - buf.release(); + payload.release(); throw e; } } - regionData.put(key, buf); + var buf = allocator.buffer(5 + payload.readableBytes()); + // vanilla region format: the length field also counts the compression type byte + buf.writeInt(payload.readableBytes() + 1); + buf.writeByte(type.index); + buf.writeBytes(payload); + payload.release(); + var previous = regionData.put(key, buf); + if (previous != null) previous.release(); } private OutputStream findCompressor(CompressType type, OutputStream out) throws IOException { @@ -82,7 +82,7 @@ public class RegionFile implements Closeable { } public void updateChunkTime(int chunkX, int chunkZ, int timestamp) { - if (chunkX > 32 || chunkZ > 32 || chunkX < 0 || chunkZ < 0) + if (chunkX >= 32 || chunkZ >= 32 || chunkX < 0 || chunkZ < 0) throw new IllegalArgumentException("Invalid chunkX+chunkZ: " + chunkX + " " + chunkZ); var index = (chunkX + 32 * chunkZ) * 4; this.timestamp.setInt(index, timestamp); @@ -105,7 +105,7 @@ public class RegionFile implements Closeable { var chunk = entry.getValue(); var size = chunk.readableBytes(); var sizeSectors = Math.ceilDiv(size, 4096); - var zeros = 4096 - (size % 4096); + var zeros = (4096 - (size % 4096)) % 4096; for (int i = 0; i < zeros; i++) { chunk.writeByte(0); } diff --git a/bundler/src/main/java/io/ib67/sfcraft/bundler/region/RegionParser.java b/bundler/src/main/java/io/ib67/sfcraft/bundler/region/RegionParser.java index 0cec9df..02ea549 100644 --- a/bundler/src/main/java/io/ib67/sfcraft/bundler/region/RegionParser.java +++ b/bundler/src/main/java/io/ib67/sfcraft/bundler/region/RegionParser.java @@ -10,8 +10,8 @@ import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.zip.DeflaterInputStream; import java.util.zip.GZIPInputStream; +import java.util.zip.InflaterInputStream; public class RegionParser implements Closeable { protected final ByteBuf source; @@ -28,7 +28,9 @@ public class RegionParser implements Closeable { try (var fc = FileChannel.open(file, StandardOpenOption.READ)) { var toRead = size; while (toRead > 0) { - toRead -= source.writeBytes(fc, 0L, (int) size); + var read = source.writeBytes(fc, (long) (size - toRead), toRead); + if (read < 0) throw new EOFException("Region file " + file + " shrank while reading"); + toRead -= read; } } } @@ -98,8 +100,9 @@ public class RegionParser implements Closeable { var key = chunks.getLong(i); var chunkXZ = readChunkXZ(key); var entry = readEntry(key); - short x = (short) (chunkXZ & 0x3F); - short z = (short) ((chunkXZ >>> 6) & 0x3F); + // packed as (x << 6) | z in readAllValidChunks + short x = (short) ((chunkXZ >>> 6) & 0x3F); + short z = (short) (chunkXZ & 0x3F); var compressType = readChunk(entry, compressed); try (var decompressor = getDecompressorByType(compressType, compressed)) { while (uncompressed.writeBytes(decompressor, 4096) != -1) ; @@ -116,7 +119,7 @@ public class RegionParser implements Closeable { private InputStream getDecompressorByType(int compressionType, ByteBuf in) throws IOException { return switch (compressionType) { case 1 -> new GZIPInputStream(new ByteBufInputStream(in)); - case 2 -> new DeflaterInputStream(new ByteBufInputStream(in)); + case 2 -> new InflaterInputStream(new ByteBufInputStream(in)); case 3 -> new ByteBufInputStream(in); case 4 -> new LZ4BlockInputStream(new ByteBufInputStream(in)); // user defined compression algorithm, not going to support. diff --git a/bundler/src/test/java/io/ib67/bundler/TestBundleReader.java b/bundler/src/test/java/io/ib67/bundler/TestBundleReader.java new file mode 100644 index 0000000..71362ad --- /dev/null +++ b/bundler/src/test/java/io/ib67/bundler/TestBundleReader.java @@ -0,0 +1,88 @@ +package io.ib67.bundler; + +import com.github.luben.zstd.ZstdOutputStreamNoFinalizer; +import io.ib67.sfcraft.bundler.BundleReader; +import io.ib67.sfcraft.bundler.BundleWriter; +import io.ib67.sfcraft.bundler.EntryOutputStream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.EOFException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestBundleReader { + + @Test + public void testChainedOverlayRestore(@TempDir Path tmp) throws IOException { + var src = Files.createDirectories(tmp.resolve("src")); + Files.writeString(src.resolve("a.txt"), "version-1-that-is-long"); + Files.writeString(src.resolve("b.txt"), "untouched"); + var fullBundle = tmp.resolve("full.swb.zst"); + BundleWriter.createBundle(fullBundle, List.of(src.resolve("a.txt"), src.resolve("b.txt")), + c -> c.relativeRoot(src)); + + // shorter content verifies the overlay truncates instead of leaving a stale tail + Files.writeString(src.resolve("a.txt"), "v2"); + var incrBundle = tmp.resolve("incr.swb.zst"); + BundleWriter.createBundle(incrBundle, List.of(src.resolve("a.txt")), + c -> c.relativeRoot(src)); + + var dest = tmp.resolve("dest"); + var reader = BundleReader.builder().build(); + reader.extract(fullBundle, dest); + reader.extract(incrBundle, dest); + + assertEquals("v2", Files.readString(dest.resolve("a.txt"))); + assertEquals("untouched", Files.readString(dest.resolve("b.txt"))); + } + + @Test + public void testZipSlipIsRejected(@TempDir Path tmp) throws IOException { + var bundle = tmp.resolve("evil.swb.zst"); + var payload = "owned".getBytes(); + try (var zstd = new ZstdOutputStreamNoFinalizer(Files.newOutputStream(bundle), 3); + var out = new EntryOutputStream(zstd)) { + out.writeEntryHeader("../evil.txt", (short) 0, payload.length, System.currentTimeMillis()); + out.write(payload); + } + var dest = tmp.resolve("dest"); + var ex = assertThrows(IOException.class, + () -> BundleReader.builder().build().extract(bundle, dest)); + assertTrue(ex.getMessage().contains("escapes"), ex.getMessage()); + assertFalse(Files.exists(tmp.resolve("evil.txt"))); + } + + @Test + public void testSymlinkIsCreatedAtLinkPath(@TempDir Path tmp) throws IOException { + var src = Files.createDirectories(tmp.resolve("src")); + var outside = Files.writeString(tmp.resolve("outside.txt"), "outside"); + var link = src.resolve("link"); + Files.createSymbolicLink(link, outside); + + var bundle = tmp.resolve("sym.swb.zst"); + BundleWriter.createBundle(bundle, List.of(link), c -> c.relativeRoot(src)); + + var dest = tmp.resolve("dest"); + BundleReader.builder().build().extract(bundle, dest); + assertTrue(Files.isSymbolicLink(dest.resolve("link")), + "the symlink itself must be recreated at the link's path, not at its target"); + } + + @Test + public void testTruncatedEntryThrowsInsteadOfLooping(@TempDir Path tmp) throws IOException { + var bundle = tmp.resolve("truncated.swb.zst"); + try (var zstd = new ZstdOutputStreamNoFinalizer(Files.newOutputStream(bundle), 3); + var out = new EntryOutputStream(zstd)) { + out.writeEntryHeader("short.bin", (short) 0, 100, System.currentTimeMillis()); + out.write(new byte[10]); // 90 bytes missing + } + var dest = tmp.resolve("dest"); + assertThrows(EOFException.class, + () -> BundleReader.builder().build().extract(bundle, dest)); + } +} diff --git a/bundler/src/test/java/io/ib67/bundler/TestRegionRoundTrip.java b/bundler/src/test/java/io/ib67/bundler/TestRegionRoundTrip.java new file mode 100644 index 0000000..838ede2 --- /dev/null +++ b/bundler/src/test/java/io/ib67/bundler/TestRegionRoundTrip.java @@ -0,0 +1,91 @@ +package io.ib67.bundler; + +import io.ib67.sfcraft.bundler.region.RegionFile; +import io.ib67.sfcraft.bundler.region.RegionParser; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; +import io.netty.buffer.Unpooled; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Simulates the full backup->restore cycle for region files: vanilla mca -> reassembled + * uncompressed (what goes into the bundle) -> reassembled ZLIB (what restore writes back), + * then verifies every chunk's uncompressed NBT is byte-identical and lives in the same + * header slot as in the original file. + */ +public class TestRegionRoundTrip { + @Test + public void roundTripPreservesEveryChunk() throws IOException { + roundTrip("region/r.9.-5.mca"); + roundTrip("region/r.-1.-1.mca"); + } + + private static void roundTrip(String resource) throws IOException { + byte[] originalBytes; + try (var in = TestRegionRoundTrip.class.getClassLoader().getResourceAsStream(resource)) { + originalBytes = in.readAllBytes(); + } + var alloc = ByteBufAllocator.DEFAULT; + + // backup side: vanilla -> NONE bundle payload + ByteBuf intermediate; + try (var parser = new RegionParser(Unpooled.wrappedBuffer(originalBytes), alloc)) { + intermediate = parser.writeReassembled(RegionFile.CompressType.NONE); + } + // restore side: bundle payload -> ZLIB region written to disk + ByteBuf restored; + try (var parser = new RegionParser(intermediate, alloc)) { + restored = parser.writeReassembled(RegionFile.CompressType.ZLIB); + } finally { + intermediate.release(); + } + + try (var originalParser = new RegionParser(Unpooled.wrappedBuffer(originalBytes), alloc); + var restoredParser = new RegionParser(restored.readerIndex(0).retain(), alloc)) { + var originalChunks = collect(originalParser); + var restoredChunks = collect(restoredParser); + assertFalse(originalChunks.isEmpty(), resource + " should contain chunks"); + assertEquals(originalChunks.keySet(), restoredChunks.keySet(), + "restored file must populate exactly the same chunk slots (no transposition)"); + for (var entry : originalChunks.entrySet()) { + var origNbt = decompress(originalParser, entry.getValue(), alloc); + var restNbt = decompress(restoredParser, restoredChunks.get(entry.getKey()), alloc); + try { + assertEquals(origNbt, restNbt, "chunk " + entry.getKey() + " must survive the round trip unchanged"); + } finally { + origNbt.release(); + restNbt.release(); + } + } + // timestamp table is copied verbatim + var origTimes = Unpooled.wrappedBuffer(originalBytes).slice(4096, 4096); + var restTimes = restored.slice(4096, 4096); + assertEquals(origTimes, restTimes, "chunk timestamps must survive the round trip"); + } finally { + restored.release(); + } + } + + /** chunkXZ -> header entry */ + private static Map collect(RegionParser parser) { + var result = new HashMap(); + var chunks = parser.readAllValidChunks(); + for (int i = 0; i < chunks.size(); i++) { + var key = chunks.getLong(i); + result.put(RegionParser.readChunkXZ(key), RegionParser.readEntry(key)); + } + return result; + } + + private static ByteBuf decompress(RegionParser parser, int entry, ByteBufAllocator alloc) throws IOException { + var buf = alloc.buffer(); + parser.readChunkDecompress(entry, buf); + return buf; + } +} diff --git a/src/main/java/io/ib67/sfcraft/Backup.java b/src/main/java/io/ib67/sfcraft/Backup.java index e08ccdb..204acd4 100644 --- a/src/main/java/io/ib67/sfcraft/Backup.java +++ b/src/main/java/io/ib67/sfcraft/Backup.java @@ -2,17 +2,41 @@ package io.ib67.sfcraft; import net.minecraft.network.chat.Component; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; + +/** + * @param subject name of the backup subject this bundle belongs to; null on entries created + * by older versions (those are treated as standalone full backups) + * @param createdAt epoch millis of creation; 0 on legacy entries + */ public record Backup( String name, String backupKey, String type, String from, + String subject, boolean incremental, + long createdAt, long sizeTotal ) { - public Component toText(){ - return Component.nullToEmpty("sizeTotal: "+Math.round(((sizeTotal / 1024.0) / 1024.0) * 100) * 0.01 + "MiB, "+ - (incremental ? "incremental" :"") + ", type: "+type+"." - ); + private static final DateTimeFormatter TIME_FORMAT = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneId.systemDefault()); + + public boolean legacy() { + return subject == null || createdAt <= 0; + } + + public Component toText() { + var size = Math.round(((sizeTotal / 1024.0) / 1024.0) * 100) * 0.01; + var sb = new StringBuilder() + .append(incremental ? "INCR" : "FULL") + .append(", ").append(size).append("MiB") + .append(", type: ").append(type); + if (createdAt > 0) { + sb.append(", at ").append(TIME_FORMAT.format(Instant.ofEpochMilli(createdAt))); + } + return Component.nullToEmpty(sb.toString()); } } diff --git a/src/main/java/io/ib67/sfcraft/BackupCommands.java b/src/main/java/io/ib67/sfcraft/BackupCommands.java index 4288ba2..ff57724 100644 --- a/src/main/java/io/ib67/sfcraft/BackupCommands.java +++ b/src/main/java/io/ib67/sfcraft/BackupCommands.java @@ -7,19 +7,20 @@ import com.mojang.brigadier.arguments.StringArgumentType; import com.mojang.brigadier.context.CommandContext; import com.mojang.brigadier.suggestion.Suggestions; import com.mojang.brigadier.suggestion.SuggestionsBuilder; +import io.ib67.sfcraft.strategy.BackupChains; import lombok.extern.log4j.Log4j2; import net.minecraft.commands.CommandBuildContext; import net.minecraft.commands.CommandSourceStack; import net.minecraft.commands.Commands; import net.minecraft.network.chat.ClickEvent; import net.minecraft.network.chat.Component; + import java.awt.*; import java.io.IOException; import java.lang.ref.WeakReference; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -import java.util.Map; import java.util.concurrent.CompletableFuture; import static net.minecraft.commands.Commands.argument; @@ -28,7 +29,7 @@ import static net.minecraft.commands.Commands.literal; @Log4j2 public class BackupCommands { protected final JustBackupMod mod; - protected Runnable restoreIssued; + protected volatile Runnable restoreIssued; protected final List helpMessage = List.of( "__ Help messages of JustBackup __", " /backup help ", @@ -37,6 +38,8 @@ public class BackupCommands { " -- List backups", " /backup restore ", " -- Restore the backup on next server restart.", + " Restoring an incremental backup automatically applies its full base", + " and every incremental in between, in order.", " Can be cancelled by /backup cancelrestore", " /backup delete ", " -- Delete a backup. This does not ask for a confirmation.", @@ -62,6 +65,7 @@ public class BackupCommands { .then(literal("help").executes(this::cmdHelp)) .then(literal("list").executes(this::cmdList)) .then(literal("cancelrestore").executes(this::cmdCancelRestore)) + .then(literal("suspend").executes(this::cmdSuspend)) .then(literal("delete") .then(argument("backupKey", StringArgumentType.greedyString()) .suggests(this::suggestBackups) @@ -74,9 +78,7 @@ public class BackupCommands { .then(argument("incremental", BoolArgumentType.bool()) .then(argument("subject", StringArgumentType.greedyString()) .executes(this::createBackupBySubject)) - .then(literal("@all").executes(this::fullBackup))) - .then(literal("suspend").executes(this::cmdSuspend)) - )); + .then(literal("@all").executes(this::fullBackup))))); } private int createBackupBySubject(CommandContext context) { @@ -94,18 +96,23 @@ public class BackupCommands { s.sendSystemMessage(Component.literal("The issued backup has been scheduled.").withColor(Color.CYAN.getRGB())); var weakRef = new WeakReference<>(s); mod.backupAll(incremental).thenAccept(result -> { - var ref = weakRef.get(); - if (ref == null) { - ref = mod.server.createCommandSourceStack(); - } - ref.sendSystemMessage(Component.literal("-- BACKUP RESULT SUMMARY --")); - for (var entry : result.entrySet()) { - var k = entry.getKey(); - var v = entry.getValue(); - ref.sendSystemMessage(Component.literal(" - [" + k + "]: ").append(v.toText())); - ref.sendSystemMessage(Component.literal("BACKUP KEY: " + v.backupKey()).withColor(Color.GRAY.getRGB())); - ref.sendSystemMessage(Component.literal("GENERATED FROM: " + v.from()).withColor(Color.GRAY.getRGB())); - } + var server = mod.server; + if (server == null || !server.isRunning()) return; + // completion runs on the backup executor; command sources must be used on the main thread + server.submit(() -> { + var ref = weakRef.get(); + if (ref == null) { + ref = server.createCommandSourceStack(); + } + ref.sendSystemMessage(Component.literal("-- BACKUP RESULT SUMMARY --")); + for (var entry : result.entrySet()) { + var k = entry.getKey(); + var v = entry.getValue(); + ref.sendSystemMessage(Component.literal(" - [" + k + "]: ").append(v.toText())); + ref.sendSystemMessage(Component.literal("BACKUP KEY: " + v.backupKey()).withColor(Color.GRAY.getRGB())); + ref.sendSystemMessage(Component.literal("GENERATED FROM: " + v.from()).withColor(Color.GRAY.getRGB())); + } + }); }); return Command.SINGLE_SUCCESS; } @@ -136,31 +143,51 @@ public class BackupCommands { s.sendSystemMessage(Component.literal("Invalid backup " + backupKey + ". Note: Use backupKey instead of backupName!")); return 0; } + final List chain; + try { + chain = BackupChains.resolveChain(mod.tracker.getTrackedBackups().values(), backup); + } catch (IllegalStateException e) { + s.sendSystemMessage(Component.literal(e.getMessage()).withColor(Color.RED.getRGB())); + return 0; + } + s.sendSystemMessage(Component.literal("The following " + chain.size() + " bundle(s) will be applied in order:")); + for (var b : chain) { + s.sendSystemMessage(Component.literal(" -> [" + (b.incremental() ? "INCR" : "FULL") + "] " + b.backupKey()) + .withColor(Color.GRAY.getRGB())); + } restoreIssued = () -> { - log.warn("EXTRACTING BACKUP " + backup + " TO " + backup.from()); - log.warn(" == STEP 1 == Make a backup for the destination."); var destination = Path.of(backup.from()); - var movedDst = destination.resolveSibling(destination.getFileName().toString()+"_bak_"+System.currentTimeMillis()); - try{ + var movedDst = destination.resolveSibling(destination.getFileName().toString() + "_bak_" + System.currentTimeMillis()); + log.warn("EXTRACTING {} BUNDLE(S) TO {}", chain.size(), destination); + log.warn(" == STEP 1 == Make a backup for the destination."); + try { Files.move(destination, movedDst); log.info("{} has been moved to {}", destination, movedDst); } catch (IOException e) { - log.error(e); - log.error("CANNOT MOVE "+destination+" TO "+movedDst); - log.error("BACKUP INTERRUPTED. Here are some tips helping you out:"); + log.error("CANNOT MOVE {} TO {}", destination, movedDst, e); + log.error("RESTORE INTERRUPTED. Here are some tips helping you out:"); log.error(" -- A. Merge already moved files"); log.error(" Try this command in your server directory: (linux)"); - log.error(" $ cp ./"+movedDst+"/* ./"+destination); + log.error(" $ cp ./{}/* ./{}", movedDst, destination); log.error(" -- B. Remove them all and use external unbundler tools"); log.error(" You may want to copy them elsewhere first (see kind A)"); - log.error(" $ rm -r ./"+destination+" ./"+movedDst); + log.error(" $ rm -r ./{} ./{}", destination, movedDst); log.error(" Visit https://github.com/saltedfishclub/justbackup for the usage of bundler tools"); log.error(" ------- ERROR END -------"); return; } - log.info(" == STEP 2 == Recovering backup "+backup); - mod.tracker.recoverBackup(backup, destination); - log.info("Backup has been recovered."); + log.info(" == STEP 2 == Applying {} bundle(s)", chain.size()); + try { + for (int i = 0; i < chain.size(); i++) { + var b = chain.get(i); + log.info("Applying {} ({}/{})", b.backupKey(), i + 1, chain.size()); + mod.tracker.recoverBackup(b, destination); + } + log.info("Backup has been recovered."); + } catch (Exception e) { + log.error("RESTORE FAILED midway. Your original world is intact at {}", movedDst, e); + log.error("Move it back to {} to undo the restore.", destination); + } }; s.sendSystemMessage(Component.literal("Backup restoration task has been scheduled!")); s.sendSystemMessage(Component.literal("Restart your server to take changes.")); @@ -180,7 +207,7 @@ public class BackupCommands { source.sendSystemMessage(Component.nullToEmpty("Backup has been deleted.")); } catch (Exception e) { source.sendSystemMessage(Component.nullToEmpty("Backup has not been deleted successfully. ERROR: " + e)); - log.error(e); + log.error("Cannot delete backup {}", backupKey, e); } return Command.SINGLE_SUCCESS; } diff --git a/src/main/java/io/ib67/sfcraft/BackupSubject.java b/src/main/java/io/ib67/sfcraft/BackupSubject.java index 766d4ef..f48e7c0 100644 --- a/src/main/java/io/ib67/sfcraft/BackupSubject.java +++ b/src/main/java/io/ib67/sfcraft/BackupSubject.java @@ -1,11 +1,64 @@ package io.ib67.sfcraft; import java.nio.file.Path; -import java.util.List; +import java.util.HashSet; +import java.util.Set; -public record BackupSubject( - String name, - Path root, - List changedFiles -) { +/** + * A directory tree being backed up, plus the set of files the watcher saw change since the + * last incremental backup. All access to the change set goes through synchronized methods so + * the watcher's adds and the worker's drain can never interleave badly (no lost events, no + * ConcurrentModificationException while bundling). + */ +public final class BackupSubject { + private final String name; + private final Path root; + private final Set changedFiles = new HashSet<>(); + private boolean forceFullNext; + + public BackupSubject(String name, Path root) { + this.name = name; + this.root = root; + } + + public String name() { + return name; + } + + public Path root() { + return root; + } + + public synchronized void addChanged(Path file) { + changedFiles.add(file); + } + + /** + * Atomically takes ownership of the accumulated change set, leaving an empty one behind. + * If the backup fails, give the files back via {@link #mergeBack(Set)}. + */ + public synchronized Set drainChanged() { + var drained = new HashSet<>(changedFiles); + changedFiles.clear(); + return drained; + } + + public synchronized void mergeBack(Set files) { + changedFiles.addAll(files); + } + + /** + * Marks the next backup as full, used when the watcher reports an OVERFLOW (events were + * dropped by the OS, so the change set can no longer be trusted). + */ + public synchronized void forceFullNext() { + forceFullNext = true; + } + + public synchronized boolean consumeForceFull() { + var v = forceFullNext; + forceFullNext = false; + if (v) changedFiles.clear(); + return v; + } } diff --git a/src/main/java/io/ib67/sfcraft/BackupWorker.java b/src/main/java/io/ib67/sfcraft/BackupWorker.java index 99187b7..52c69a5 100644 --- a/src/main/java/io/ib67/sfcraft/BackupWorker.java +++ b/src/main/java/io/ib67/sfcraft/BackupWorker.java @@ -6,21 +6,45 @@ import lombok.SneakyThrows; import lombok.extern.log4j.Log4j2; import net.minecraft.server.MinecraftServer; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Supplier; import java.util.function.UnaryOperator; +import java.util.stream.Stream; -import static io.ib67.sfcraft.Globals.BACKUP_LOCK; +import static io.ib67.sfcraft.Globals.WORLD_IO_LOCK; +/** + * Runs on the backup executor thread. Flow: + *
    + *
  1. Save the world on the main thread (saveEverything is not thread-safe anywhere else) + * with flush=true, draining the IOWorkers so region files are complete on disk.
    2. + *
  2. Acquire the write side of {@link Globals#WORLD_IO_LOCK}, parking all region IO.
    3. + *
  3. Reflink-clone the files to a staging dir and release the lock early (CoW + * filesystems), or keep holding it while packing (fallback).
    4. + *
  4. Hand the bundle to the storage strategy.
    5. + *
+ * The main thread never blocks on the lock — autoSave/saveEverything are skipped while it is + * write-locked (see MixinMinecraftServer) — so waiting for main-thread tasks while holding + * the write lock cannot deadlock. + */ @Log4j2 public record BackupWorker( MinecraftServer server, BackupStrategy strategy, Path tmpDir, BackupSubject subject, + boolean incremental, + List excludedRoots, UnaryOperator config ) implements Supplier { + private static final long LOCK_TIMEOUT_SECONDS = 60; @SneakyThrows public BackupWorker { @@ -33,30 +57,116 @@ public record BackupWorker( if (server == null) { throw new IllegalStateException("MinecraftServer is not initialized yet, not doing backup"); } - server.saveEverything(true, true, false); - while (!BACKUP_LOCK.compareAndSet(IOState.IDLE, IOState.BACKUP)) { - log.info("Server is saving world while performing backup! waiting..."); - Thread.sleep(2000); + var createdAt = System.currentTimeMillis(); + var full = !incremental || subject.consumeForceFull(); + + runOnMainThread(() -> server.saveEverything(true, true, false), "world save"); + + long stamp = WORLD_IO_LOCK.tryWriteLock(LOCK_TIMEOUT_SECONDS, TimeUnit.SECONDS); + if (stamp == 0) { + throw new IllegalStateException("World IO is still busy after " + LOCK_TIMEOUT_SECONDS + + "s, aborting backup of " + subject.name()); } - // performing backup is now true - var tmpBundle = tmpDir.resolve("backup_" + subject.name() + "_" + System.currentTimeMillis() + ".swb.zst"); + + var tmpBundle = tmpDir.resolve("backup_" + subject.name() + "_" + createdAt + ".swb.zst"); + Set drained = null; + Path stagingRoot = null; try { - // tmp file path - log.info("Creating bundle {}", tmpBundle); - BundleWriter.createBundle(tmpBundle, subject.changedFiles(), config); + // Barrier: a main-thread save that started before we locked has finished by now. + runOnMainThread(() -> { + }, "main-thread barrier"); + + List files; + if (full) { + files = collectFullFileList(); + } else { + drained = subject.drainChanged(); + files = drained.stream().filter(this::eligible).toList(); + } + + var packRoot = subject.root(); + stagingRoot = ReflinkCloner.tryStage(subject.name(), packRoot, files, + tmpDir.resolve("staging").resolve(subject.name() + "_" + createdAt)); + if (stagingRoot != null) { + var absRoot = packRoot.toAbsolutePath().normalize(); + var absStaging = stagingRoot; + files = files.stream() + .map(f -> absStaging.resolve(absRoot.relativize(f.toAbsolutePath().normalize()))) + .toList(); + packRoot = stagingRoot; + WORLD_IO_LOCK.unlockWrite(stamp); + stamp = 0; + log.info("Reflinked {} files to staging; world IO resumed, packing without the lock.", files.size()); + } + + log.info("Creating {} bundle {} ({} files)", full ? "full" : "incremental", tmpBundle, files.size()); + var finalPackRoot = packRoot; + BundleWriter.createBundle(tmpBundle, files, c -> config.apply(c).relativeRoot(finalPackRoot)); } catch (Exception e) { + if (drained != null) subject.mergeBack(drained); + Files.deleteIfExists(tmpBundle); throw e; } finally { - // Though we won't access the state within backup progress, there are certain cases - // that this state will be changed to STORAGE_SYNC. - // Also take a look at MixinRegionBasedStorage. - while (!BACKUP_LOCK.compareAndSet(IOState.BACKUP, IOState.IDLE)) { - log.error("Expect BACKUP state but got {}", BACKUP_LOCK.get()); - Thread.sleep(2000); + if (stamp != 0) WORLD_IO_LOCK.unlockWrite(stamp); + if (stagingRoot != null) { + try { + FsUtil.deleteRecursively(stagingRoot); + } catch (IOException e) { + log.warn("Cannot clean up staging dir {}", stagingRoot, e); + } } } - var result = strategy.createBackup(subject.name(), subject.root(), tmpBundle); - Files.deleteIfExists(tmpBundle); - return result; + + try { + return strategy.createBackup(subject.name(), subject.root(), tmpBundle, !full, createdAt); + } catch (Exception e) { + if (drained != null) subject.mergeBack(drained); + throw e; + } finally { + Files.deleteIfExists(tmpBundle); + } + } + + /** + * Runs a task on the main thread, but never hangs: once the server leaves its tick loop + * (shutdown) submitted tasks are no longer processed, so we poll and abort the backup + * instead of blocking forever. + */ + private void runOnMainThread(Runnable task, String what) { + var future = server.submit(task); + while (true) { + try { + future.get(1, TimeUnit.SECONDS); + return; + } catch (TimeoutException e) { + if (!server.isRunning()) { + future.cancel(false); + throw new IllegalStateException("Server is shutting down, aborting backup while waiting for " + what); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for " + what); + } catch (ExecutionException e) { + throw new IllegalStateException(what + " failed", e.getCause()); + } + } + } + + @SneakyThrows + private List collectFullFileList() { + try (Stream s = Files.walk(subject.root())) { + return s.filter(Files::isRegularFile) + .filter(this::eligible) + .toList(); + } + } + + private boolean eligible(Path path) { + if (path.getFileName().toString().equals("session.lock")) return false; + var abs = path.toAbsolutePath().normalize(); + for (Path excluded : excludedRoots) { + if (abs.startsWith(excluded.toAbsolutePath().normalize())) return false; + } + return true; } } diff --git a/src/main/java/io/ib67/sfcraft/FileWatcherThread.java b/src/main/java/io/ib67/sfcraft/FileWatcherThread.java index 4ea8b28..5cec25f 100644 --- a/src/main/java/io/ib67/sfcraft/FileWatcherThread.java +++ b/src/main/java/io/ib67/sfcraft/FileWatcherThread.java @@ -3,87 +3,181 @@ package io.ib67.sfcraft; import lombok.SneakyThrows; import lombok.extern.log4j.Log4j2; +import java.io.IOException; import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; import java.util.HashMap; import java.util.Map; -import java.util.Objects; import java.util.concurrent.TimeUnit; -import java.util.function.BiConsumer; -import java.util.stream.Collectors; import java.util.stream.Stream; @Log4j2 public class FileWatcherThread extends Thread { - protected final Map watchSubject; - protected final BiConsumer eventListener; + private static final long VALIDATE_INTERVAL_MILLIS = 10_000; - public FileWatcherThread( - Map watchSubject, - BiConsumer eventListener - ) { - this.watchSubject = watchSubject.entrySet() - .stream() - // todo support for nonexist dir - .flatMap(it -> walkThrough(it.getValue()).map(r -> Map.entry(r.toAbsolutePath(), it.getKey()))) - .filter(it->Files.isDirectory(it.getKey())) - .collect(Collectors.toMap( - Map.Entry::getKey, - Map.Entry::getValue - )); - this.eventListener = eventListener; - setName("JustBackup/FileWatcher"); - setDaemon(true); + public interface Listener { + void onChange(String subject, Path file); + + /** + * Events for this subject were lost (OS queue overflow, or its directory tree was + * replaced underneath us); the accumulated change set can no longer be trusted and + * the next backup should be a full one. + */ + void onOverflow(String subject); } + /** subject name -> root directory (absolute) */ + protected final Map subjectRoots = new HashMap<>(); + /** subject name -> fileKey (inode) of the root at registration time */ + protected final Map subjectRootKeys = new HashMap<>(); + /** watched directory (absolute) -> subject name */ + protected final Map watchedDirs = new HashMap<>(); + protected final Map keyToDir = new HashMap<>(); + protected final Listener listener; + private WatchService watcher; + private long lastValidation; + @SneakyThrows - private static Stream walkThrough(Path path) { - if(Files.notExists(path)){ - log.error("{} not found. Creating directory for it..", path); - Files.createDirectories(path); + public FileWatcherThread( + Map watchSubject, + Listener listener + ) { + for (var entry : watchSubject.entrySet()) { + var root = entry.getValue(); + if (Files.notExists(root)) { + log.error("{} not found. Creating directory for it..", root); + Files.createDirectories(root); + } + subjectRoots.put(entry.getKey(), root.toAbsolutePath()); } - return Files.walk(path); + this.listener = listener; + setName("JustBackup/FileWatcher"); + setDaemon(true); } @Override @SneakyThrows public void run() { - try (var watcher = FileSystems.getDefault().newWatchService()) { - var keyToParent = new HashMap(); - for (Path value : watchSubject.keySet()) { - var key = value.register( - watcher, - StandardWatchEventKinds.ENTRY_CREATE, - StandardWatchEventKinds.ENTRY_MODIFY - ); - keyToParent.put(key, value); + try (var watchService = FileSystems.getDefault().newWatchService()) { + this.watcher = watchService; + for (var entry : subjectRoots.entrySet()) { + registerSubject(entry.getKey(), false); } + log.info("Watching {} directories for {} subjects", keyToDir.size(), subjectRoots.size()); + lastValidation = System.currentTimeMillis(); for (; ; ) { var key = watcher.poll(10, TimeUnit.SECONDS); - if (key == null) continue; if (Thread.interrupted()) return; + if (System.currentTimeMillis() - lastValidation >= VALIDATE_INTERVAL_MILLIS) { + validateRoots(); + } + if (key == null) continue; + var dir = keyToDir.get(key); + var subject = dir == null ? null : watchedDirs.get(dir); for (var ev : key.pollEvents()) { - var kind = ev.kind(); - if (kind == StandardWatchEventKinds.OVERFLOW) - continue; - var parent = keyToParent.get(key); - var subject = watchSubject.get(parent); - if(subject == null) { - log.warn("Ignoring changes in "+parent); + if (subject == null) { + log.warn("Ignoring changes in unknown watch key {}", dir); continue; } - var path = parent.resolve(((WatchEvent) ev).context()); + var kind = ev.kind(); + if (kind == StandardWatchEventKinds.OVERFLOW) { + log.warn("Watch events for {} overflowed; forcing a full backup on next run.", subject); + listener.onOverflow(subject); + continue; + } + var path = dir.resolve((Path) ev.context()); + log.debug("event {} on {} ({})", kind, path, subject); if (Files.isRegularFile(path)) { - eventListener.accept(subject, parent); + listener.onChange(subject, path); } else if (Files.isDirectory(path) && kind == StandardWatchEventKinds.ENTRY_CREATE) { - var nK = path.register(watcher, StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_MODIFY); - watchSubject.put(path, subject); - keyToParent.put(nK, path); + // report contained files too: they may have been created before we + // managed to register and would never produce their own events + registerTree(path, subject, true); } } - key.reset(); + if (!key.reset()) { + log.debug("Watch key for {} is no longer valid, dropping it", dir); + keyToDir.remove(key); + if (dir != null) watchedDirs.remove(dir); + } } - }catch (InterruptedException interruptedException){ + } catch (InterruptedException interruptedException) { log.info("FileWatcherThread is quitting!"); } } + + /** + * Watch keys silently go stale when a watched root is replaced rather than modified — + * e.g. a world format conversion moves the directory aside and recreates it, leaving us + * watching an orphaned inode. Compare the root's fileKey (inode) against what we + * registered; on mismatch, re-register the whole subject and force a full backup since + * events were lost in between. + */ + private void validateRoots() { + lastValidation = System.currentTimeMillis(); + for (var entry : subjectRoots.entrySet()) { + var subject = entry.getKey(); + var root = entry.getValue(); + Object currentKey = fileKeyOf(root); + if (java.util.Objects.equals(currentKey, subjectRootKeys.get(subject))) continue; + log.warn("Root of subject {} ({}) was replaced or recreated; re-registering watches and forcing a full backup.", + subject, root); + registerSubject(subject, true); + } + } + + private static Object fileKeyOf(Path path) { + try { + return Files.readAttributes(path, BasicFileAttributes.class).fileKey(); + } catch (IOException e) { + return null; + } + } + + /** + * (Re-)registers a subject's whole directory tree. Existing watches of the subject are + * cancelled first; when {@code lostEvents} is set, the subject is flagged for a full + * backup because changes may have happened while we were not watching. + */ + private void registerSubject(String subject, boolean lostEvents) { + keyToDir.entrySet().removeIf(e -> { + if (subject.equals(watchedDirs.get(e.getValue()))) { + e.getKey().cancel(); + return true; + } + return false; + }); + watchedDirs.values().removeIf(subject::equals); + + var root = subjectRoots.get(subject); + subjectRootKeys.put(subject, fileKeyOf(root)); + if (Files.isDirectory(root)) { + // no need to report existing files: the startup full backup (initial call) or the + // forced full backup (lostEvents) already covers them + registerTree(root, subject, false); + } else { + log.error("Root of subject {} ({}) does not exist, nothing is watched for it.", subject, root); + } + if (lostEvents) listener.onOverflow(subject); + } + + private void registerTree(Path root, String subject, boolean reportFiles) { + try (Stream walk = Files.walk(root)) { + for (var p : walk.toList()) { + if (Files.isDirectory(p)) { + var abs = p.toAbsolutePath(); + var key = abs.register(watcher, + StandardWatchEventKinds.ENTRY_CREATE, + StandardWatchEventKinds.ENTRY_MODIFY); + watchedDirs.put(abs, subject); + keyToDir.put(key, abs); + } else if (reportFiles && Files.isRegularFile(p)) { + listener.onChange(subject, p); + } + } + } catch (IOException e) { + log.warn("Cannot watch directory tree {}; forcing a full backup on next run.", root, e); + listener.onOverflow(subject); + } + } } diff --git a/src/main/java/io/ib67/sfcraft/FsUtil.java b/src/main/java/io/ib67/sfcraft/FsUtil.java new file mode 100644 index 0000000..d1fdce4 --- /dev/null +++ b/src/main/java/io/ib67/sfcraft/FsUtil.java @@ -0,0 +1,31 @@ +package io.ib67.sfcraft; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; + +public final class FsUtil { + private FsUtil() { + } + + public static void deleteRecursively(Path root) throws IOException { + if (Files.notExists(root)) return; + Files.walkFileTree(root, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.deleteIfExists(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + if (exc != null) throw exc; + Files.deleteIfExists(dir); + return FileVisitResult.CONTINUE; + } + }); + } +} diff --git a/src/main/java/io/ib67/sfcraft/Globals.java b/src/main/java/io/ib67/sfcraft/Globals.java index 8062353..354d5b6 100644 --- a/src/main/java/io/ib67/sfcraft/Globals.java +++ b/src/main/java/io/ib67/sfcraft/Globals.java @@ -6,18 +6,26 @@ import io.ib67.sfcraft.config.StorageOption; import io.ib67.sfcraft.config.serializer.StorageOptionSerializer; import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.StampedLock; public class Globals { /** - * for api users: DO NOT MUTATE THIS STATE IN WHATEVER CASE SINCE THIS IS CAUTIOUSLY ALIGNED WITH THE CURRENT THREAD MODEL - * AND DOING SO WILL BREAK YOUR WORLD - * This state indicates: - * 1. A minecraft auto-save is in-progress. - * 2. The backup is in-progress - * 3. Some chunks are writing to disk + * Guards on-disk world files during backups. + *

+ * Chunk IO ({@code RegionFileStorage.write/flush}, running on the per-dimension IOWorker + * threads) holds the read lock; the backup worker holds the write lock while it snapshots + * files. The main thread never blocks on this lock: {@code autoSave} and manual + * {@code saveEverything} calls are skipped while the write lock is held (see + * {@code MixinMinecraftServer}), which is what makes it safe for the backup thread to + * wait on main-thread tasks while holding the write lock. + *

+ * A {@link StampedLock} is used instead of a ReentrantReadWriteLock because the write + * stamp may be released from a different code path than the one that acquired it + * (early release once files are cloned to staging). None of the guarded sections are + * reentrant. */ - public static final AtomicReference BACKUP_LOCK = new AtomicReference<>(IOState.IDLE); + public static final StampedLock WORLD_IO_LOCK = new StampedLock(); + public static final Gson GSON = new GsonBuilder().setPrettyPrinting() .registerTypeAdapter(StorageOption.class, new StorageOptionSerializer( Map.of("s3", StorageOption.S3.class, diff --git a/src/main/java/io/ib67/sfcraft/IOState.java b/src/main/java/io/ib67/sfcraft/IOState.java deleted file mode 100644 index f67c8f1..0000000 --- a/src/main/java/io/ib67/sfcraft/IOState.java +++ /dev/null @@ -1,8 +0,0 @@ -package io.ib67.sfcraft; - -public enum IOState { - SAVING_WORLD, - BACKUP, - STORAGE_SYNC, - IDLE -} diff --git a/src/main/java/io/ib67/sfcraft/JustBackupMod.java b/src/main/java/io/ib67/sfcraft/JustBackupMod.java index 88e7e6e..9f886d7 100644 --- a/src/main/java/io/ib67/sfcraft/JustBackupMod.java +++ b/src/main/java/io/ib67/sfcraft/JustBackupMod.java @@ -4,6 +4,7 @@ import com.github.luben.zstd.Zstd; import com.google.gson.reflect.TypeToken; import io.ib67.sfcraft.config.JustBackupConfig; import io.ib67.sfcraft.config.StorageOption; +import io.ib67.sfcraft.strategy.BackupChains; import io.ib67.sfcraft.strategy.BackupStrategy; import io.ib67.sfcraft.strategy.BackupTracker; import io.ib67.sfcraft.strategy.LocalBackupStrategy; @@ -25,14 +26,16 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; -public class JustBackupMod implements ModInitializer { +public class JustBackupMod implements ModInitializer, FileWatcherThread.Listener { public static final String MOD_ID = "justbackup"; public static final Logger LOGGER = LoggerFactory.getLogger(MOD_ID); protected final Map backupSubjects = new HashMap<>(); @@ -44,6 +47,8 @@ public class JustBackupMod implements ModInitializer { protected JustBackupConfig config; protected BackupTracker tracker; protected Path backupIndexPath; + protected Path tmpDir; + protected List excludedRoots; protected FileWatcherThread watcherThread; @Override @@ -62,33 +67,45 @@ public class JustBackupMod implements ModInitializer { @SneakyThrows private void doInitialize() { LOGGER.info("Loading backup configuration"); - Path config1 = Path.of("config"); - backupIndexPath = config1.resolve("just_backup_index.json"); - var configPath = config1.resolve("just_backup.json"); + Path configDir = Path.of("config"); + backupIndexPath = configDir.resolve("just_backup_index.json"); + var configPath = configDir.resolve("just_backup.json"); if (Files.notExists(configPath.getParent())) Files.createDirectories(configPath.getParent()); if (Files.notExists(configPath)) saveConfig(configPath); - if (Files.notExists(backupIndexPath)) - Files.createFile(backupIndexPath); config = Globals.GSON.fromJson(Files.readString(configPath), JustBackupConfig.class); config.backupSubjects().forEach((k, v) -> { var subjectPath = Path.of(v); if (Files.notExists(subjectPath)) { - LOGGER.error("Backup _subject {} does not exist.", v); + LOGGER.error("Backup subject {} does not exist.", v); return; } - backupSubjects.put(k, new BackupSubject(k, subjectPath, new ArrayList<>())); + backupSubjects.put(k, new BackupSubject(k, subjectPath)); }); if (config.zstdDictPath() != null && !config.zstdDictPath().isEmpty()) { zstdDict = Files.readAllBytes(Path.of(config.zstdDictPath())); } + tmpDir = Path.of(config.temporaryBackupDir()); + excludedRoots = new ArrayList<>(); + excludedRoots.add(tmpDir); + if (config.option() instanceof StorageOption.Local local) { + excludedRoots.add(Path.of(local.saveDir())); + } + var strategy = createStrategy(); - this.tracker = new BackupTracker(strategy, Globals.GSON.fromJson(Files.readString(backupIndexPath), new TypeToken<>() { - })); + Map initialIndex = Files.exists(backupIndexPath) + ? Globals.GSON.fromJson(Files.readString(backupIndexPath), new TypeToken>() { + }) + : null; + this.tracker = new BackupTracker(strategy, initialIndex, backupIndexPath); LOGGER.info("Configuration has been loaded. Using storage option: {}", config.option().type()); - scheduledBackupExecutor = Executors.newSingleThreadScheduledExecutor(); + scheduledBackupExecutor = Executors.newSingleThreadScheduledExecutor(r -> { + var t = new Thread(r, "JustBackup/Worker"); + t.setDaemon(false); + return t; + }); registerWatchService(); registerHandlers(); } @@ -98,7 +115,7 @@ public class JustBackupMod implements ModInitializer { config.backupSubjects().entrySet().stream() .map(it -> Map.entry(it.getKey(), Path.of(it.getValue()))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)), - this::handleFileChanges + this ); Runtime.getRuntime().addShutdownHook(new Thread(() -> { if (watcherThread.getState() != Thread.State.TERMINATED) watcherThread.interrupt(); @@ -106,12 +123,16 @@ public class JustBackupMod implements ModInitializer { watcherThread.start(); } - private void handleFileChanges(String _subject, Path path) { - synchronized (backupSubjects) { - var subject = backupSubjects.get(_subject); - if (subject != null) - subject.changedFiles().add(path); // check null - } + @Override + public void onChange(String subjectName, Path file) { + var subject = backupSubjects.get(subjectName); + if (subject != null) subject.addChanged(file); + } + + @Override + public void onOverflow(String subjectName) { + var subject = backupSubjects.get(subjectName); + if (subject != null) subject.forceFullNext(); } @SneakyThrows @@ -130,96 +151,168 @@ public class JustBackupMod implements ModInitializer { } lastBackupServerTicks = server.getTickCount(); if (!suspend) backupAll(config.incremental()).thenAccept(it -> { - LOGGER.info("Backup success!"); - it.forEach((k, v) -> LOGGER.info(k + ": " + v)); + LOGGER.info("Scheduled backup finished."); + it.forEach((k, v) -> LOGGER.info("{}: {}", k, v)); }); }, config.backupIntervalMinutes(), config.backupIntervalMinutes(), TimeUnit.MINUTES); }); - ServerLifecycleEvents.SERVER_STOPPED.register(server -> { - try { - Files.deleteIfExists(Path.of(config.temporaryBackupDir())); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); var command = new BackupCommands(this); CommandRegistrationCallback.EVENT.register(command::registerCommand); - ServerLifecycleEvents.SERVER_STOPPED.register(server -> { - LOGGER.info("Shutting down the executor"); + // Runs at the head of stopServer, before the final world save: in-flight backups must + // release the world IO lock before that save, or the flush would park forever. + ServerLifecycleEvents.SERVER_STOPPING.register(server -> { + suspend = true; scheduledBackupExecutor.shutdown(); - if (command.restoreIssued != null) { - command.restoreIssued.run(); + try { + if (!scheduledBackupExecutor.awaitTermination(15, TimeUnit.SECONDS)) { + LOGGER.info("Waiting up to 10 minutes for the in-flight backup to finish..."); + if (!scheduledBackupExecutor.awaitTermination(10, TimeUnit.MINUTES)) { + LOGGER.error("In-flight backup did not finish in time; its bundle may be incomplete."); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); } - tracker.close(); }); + ServerLifecycleEvents.SERVER_STOPPED.register(server -> { + if (command.restoreIssued != null) { + try { + command.restoreIssued.run(); + } catch (Exception e) { + LOGGER.error("Backup restoration failed", e); + } + } + try { + tracker.close(); + } catch (Exception e) { + LOGGER.error("Error while closing the backup storage", e); + } + cleanTemporaryDir(); + if (watcherThread != null && watcherThread.getState() != Thread.State.TERMINATED) { + watcherThread.interrupt(); + } + }); + } + + /** + * Removes only artifacts we created (staging clones, leftover bundles, temp downloads). + * Never deletes the directory itself: a misconfigured temporaryBackupDir pointing at the + * server root must not wipe the server. + */ + private void cleanTemporaryDir() { + try { + FsUtil.deleteRecursively(tmpDir.resolve("staging")); + } catch (IOException e) { + LOGGER.warn("Cannot clean staging dir", e); + } + if (Files.notExists(tmpDir)) return; + try (Stream files = Files.list(tmpDir)) { + files.filter(Files::isRegularFile) + .filter(p -> { + var n = p.getFileName().toString(); + return (n.startsWith("backup_") && n.endsWith(".swb.zst")) || n.startsWith("s3_"); + }) + .forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException e) { + LOGGER.warn("Cannot delete leftover {}", p, e); + } + }); + } catch (IOException e) { + LOGGER.warn("Cannot clean temporary backup dir", e); + } + } + + /** + * Logs the message and, when enabled and a server is up, broadcasts it to players from + * the main thread (PlayerList is not safe to touch from the backup executor). + */ + public void broadcast(Component message) { + LOGGER.info(message.getString()); + if (!config.broadcast()) return; + var s = server; + if (s == null || !s.isRunning()) return; + s.submit(() -> s.getPlayerList().broadcastSystemMessage(message, false)); } public CompletableFuture> backupAll(boolean incremental) { if (backupSubjects.isEmpty()) return CompletableFuture.completedFuture(Map.of()); var map = new HashMap>(); - server.getPlayerList().broadcastSystemMessage( - Component.literal(" BACKUP >> ").withColor(Color.RED.getRGB()).withStyle(it -> it.withBold(true)) - .append(Component.nullToEmpty("The server is performing a backup, you may experience some lag.")), - false - ); + broadcast(Component.literal(" BACKUP >> ").withColor(Color.RED.getRGB()).withStyle(it -> it.withBold(true)) + .append(Component.nullToEmpty("The server is performing a backup, you may experience some lag."))); for (var entry : backupSubjects.entrySet()) { map.put(entry.getKey(), issueBackup(entry.getKey(), incremental)); } return CompletableFuture.allOf(map.values().toArray(new CompletableFuture[0])) .thenApply(it -> { - var _map = new HashMap(); + var results = new HashMap(); map.forEach((k, v) -> { var result = v.join(); LOGGER.info("Backup information of {}: {}", k, result); - _map.put(k, result); + results.put(k, result); }); - return _map; + return results; }); } public CompletableFuture issueBackup(String subjectName, boolean incremental) { var subject = backupSubjects.get(subjectName); if (subject == null) - return CompletableFuture.failedFuture(new IllegalArgumentException("Unknown backup _subject: " + subjectName)); - if (!incremental) { - try (var s = Files.walk(subject.root())) { - subject = new BackupSubject(subject.name(), subject.root(), s.filter(Files::isRegularFile) - .filter(it -> !it.getFileName().toString().equals("session.lock")) - .toList()); - } catch (IOException e) { - return CompletableFuture.failedFuture(new IllegalArgumentException("Cannot perform full backup for " + subjectName, e)); - } - } - var finalSubject = subject; + return CompletableFuture.failedFuture(new IllegalArgumentException("Unknown backup subject: " + subjectName)); if (config.allowGunzip()) LOGGER.info("Chunk reassembler is enabled!"); - var worker = new BackupWorker(server, tracker, Path.of(config.temporaryBackupDir()), - subject, c -> + var worker = new BackupWorker(server, tracker, tmpDir, subject, incremental, excludedRoots, c -> c.allowGunzip(config.allowGunzip()) - .relativeRoot(finalSubject.root()) .allocator(ByteBufAllocator.DEFAULT) .maxWorkers(config.compressionWorkerThreads()) .compressionLevel(config.compressionLevel()) .dictionary(zstdDict) ); - return CompletableFuture.supplyAsync(worker, scheduledBackupExecutor).whenComplete(this::onBackupComplete); + return CompletableFuture.supplyAsync(() -> { + if (!tracker.isAvailable()) { + throw new IllegalStateException("Backup storage is not available (quota exceeded?), skipping backup of " + subjectName); + } + return worker.get(); + }, scheduledBackupExecutor).whenComplete(this::onBackupComplete); } - @SneakyThrows private void onBackupComplete(@UnknownNullability Backup backup, @UnknownNullability Throwable throwable) { if (throwable != null) { - server.getPlayerList().broadcastSystemMessage(Component.literal("Backup failed. For administrators, please check your server console."), false); - LOGGER.error(throwable.getMessage(), throwable); + broadcast(Component.literal("Backup failed. For administrators, please check your server console.")); + LOGGER.error("Backup failed", throwable); } else { - Files.writeString(backupIndexPath, Globals.GSON.toJson(tracker.getTrackedBackups())); - server.getPlayerList().broadcastSystemMessage(Component.literal("Backup success. " + backup), false); + broadcast(Component.literal("Backup success. ").append(backup.toText())); + rotate(backup.subject()); + } + } + + /** + * Deletes whole generations (a full backup plus its incrementals) beyond the newest + * {@code keepBackups} ones. Entries without chain metadata are never rotated away. + */ + private void rotate(String subjectName) { + if (subjectName == null) return; + try { + var victims = BackupChains.rotationVictims( + tracker.getTrackedBackups().values(), subjectName, config.keepBackups()); + for (var victim : victims) { + LOGGER.info("Rotating out old backup {} ({})", victim.backupKey(), victim.incremental() ? "INCR" : "FULL"); + try { + tracker.deleteBackup(victim); + } catch (Exception e) { + LOGGER.warn("Cannot delete old backup {}", victim.backupKey(), e); + } + } + } catch (Exception e) { + LOGGER.warn("Backup rotation failed for {}", subjectName, e); } } private BackupStrategy createStrategy() { return switch (config.option()) { case StorageOption.Local local -> new LocalBackupStrategy(local); - case StorageOption.S3 s3 -> new S3BackupStrategy(s3, Path.of(config.temporaryBackupDir())); + case StorageOption.S3 s3 -> new S3BackupStrategy(s3, tmpDir); }; } @@ -245,4 +338,4 @@ public class JustBackupMod implements ModInitializer { } Files.writeString(configPath, Globals.GSON.toJson(cfg)); } -} \ No newline at end of file +} diff --git a/src/main/java/io/ib67/sfcraft/ReflinkCloner.java b/src/main/java/io/ib67/sfcraft/ReflinkCloner.java new file mode 100644 index 0000000..a345df0 --- /dev/null +++ b/src/main/java/io/ib67/sfcraft/ReflinkCloner.java @@ -0,0 +1,163 @@ +package io.ib67.sfcraft; + +import lombok.extern.log4j.Log4j2; + +import java.lang.foreign.Arena; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemoryLayout; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.StructLayout; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.VarHandle; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Clones files via the Linux {@code FICLONE} ioctl (reflink). On CoW filesystems + * (btrfs, XFS with reflink, OpenZFS >= 2.2) this is a metadata-only operation: the clone + * shares extents with the source, so staging a whole backup takes milliseconds and no + * extra disk space, letting the backup release the world IO lock before compressing. + *

+ * Unsupported setups (non-Linux, ext4, overlayfs, staging dir on a different mount) are + * detected on first use and cached per subject; callers then fall back to packing while + * holding the world IO lock. + */ +@Log4j2 +public final class ReflinkCloner { + private static final long FICLONE = 0x40049409L; + private static final int O_RDONLY = 0; + private static final int O_WRONLY = 1; + private static final int O_CREAT = 0x40; + private static final int O_TRUNC = 0x200; + private static final int ENOENT = 2; + private static final int EXDEV = 18; + + private static final MethodHandle OPEN; + private static final MethodHandle IOCTL; + private static final MethodHandle CLOSE; + private static final StructLayout CAPTURE_LAYOUT; + private static final VarHandle ERRNO; + private static final boolean AVAILABLE; + private static final Set UNSUPPORTED_SUBJECTS = ConcurrentHashMap.newKeySet(); + + static { + MethodHandle open = null, ioctl = null, close = null; + StructLayout capture = null; + VarHandle errno = null; + boolean ok = false; + try { + if (System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("linux")) { + var linker = Linker.nativeLinker(); + var lookup = linker.defaultLookup(); + var captureErrno = Linker.Option.captureCallState("errno"); + capture = Linker.Option.captureStateLayout(); + errno = capture.varHandle(MemoryLayout.PathElement.groupElement("errno")); + open = linker.downcallHandle(lookup.findOrThrow("open"), + FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.ADDRESS, ValueLayout.JAVA_INT, ValueLayout.JAVA_INT), + captureErrno); + ioctl = linker.downcallHandle(lookup.findOrThrow("ioctl"), + FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG, ValueLayout.JAVA_INT), + captureErrno, Linker.Option.firstVariadicArg(2)); + close = linker.downcallHandle(lookup.findOrThrow("close"), + FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_INT)); + ok = true; + } else { + log.info("Reflink fast path is Linux-only; backups will pack while holding the world IO lock."); + } + } catch (Throwable t) { + log.info("Reflink fast path unavailable ({}); backups will pack while holding the world IO lock.", t.toString()); + } + OPEN = open; + IOCTL = ioctl; + CLOSE = close; + CAPTURE_LAYOUT = capture; + ERRNO = errno; + AVAILABLE = ok; + } + + private ReflinkCloner() { + } + + /** + * Clones {@code files} (which must live under {@code subjectRoot}) into + * {@code stagingRoot}, preserving their relative structure. Files that vanished since + * being listed are skipped. + * + * @return the staging root on success, or null if reflink is unsupported here — in that + * case any partial staging has been cleaned up and the subject is remembered as + * unsupported so future backups skip the attempt. + */ + public static Path tryStage(String subjectName, Path subjectRoot, List files, Path stagingRoot) { + if (!AVAILABLE || UNSUPPORTED_SUBJECTS.contains(subjectName)) return null; + var absRoot = subjectRoot.toAbsolutePath().normalize(); + try { + Files.createDirectories(stagingRoot); + for (Path file : files) { + var src = file.toAbsolutePath().normalize(); + if (!Files.isRegularFile(src)) continue; + var dst = stagingRoot.resolve(absRoot.relativize(src)); + Files.createDirectories(dst.getParent()); + int err = clone(src, dst); + if (err == ENOENT) continue; // deleted between listing and cloning + if (err != 0) { + markUnsupported(subjectName, src, err); + FsUtil.deleteRecursively(stagingRoot); + return null; + } + } + return stagingRoot; + } catch (Throwable t) { + log.warn("Reflink staging failed; falling back to packing under the world IO lock.", t); + UNSUPPORTED_SUBJECTS.add(subjectName); + try { + FsUtil.deleteRecursively(stagingRoot); + } catch (Exception cleanup) { + log.warn("Cannot clean up staging dir {}", stagingRoot, cleanup); + } + return null; + } + } + + private static void markUnsupported(String subjectName, Path src, int err) { + UNSUPPORTED_SUBJECTS.add(subjectName); + if (err == EXDEV) { + log.warn("Reflink not possible for subject {}: {} and the temporary backup dir are on " + + "different filesystems. Put temporaryBackupDir on the same filesystem as the " + + "backup subject to enable the CoW fast path. Falling back to packing under the world IO lock.", subjectName, src); + } else { + log.info("Reflink unsupported for subject {} (errno {} on {}); " + + "falling back to packing under the world IO lock.", subjectName, err, src); + } + } + + private static int clone(Path src, Path dst) throws Throwable { + try (var arena = Arena.ofConfined()) { + var capture = arena.allocate(CAPTURE_LAYOUT); + int srcFd = (int) OPEN.invokeExact(capture, arena.allocateFrom(src.toString()), O_RDONLY, 0); + if (srcFd < 0) return errno(capture); + try { + int dstFd = (int) OPEN.invokeExact(capture, arena.allocateFrom(dst.toString()), + O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (dstFd < 0) return errno(capture); + try { + int result = (int) IOCTL.invokeExact(capture, dstFd, FICLONE, srcFd); + return result == 0 ? 0 : errno(capture); + } finally { + var ignored = (int) CLOSE.invokeExact(dstFd); + } + } finally { + var ignored = (int) CLOSE.invokeExact(srcFd); + } + } + } + + private static int errno(MemorySegment capture) { + return (int) ERRNO.get(capture, 0L); + } +} diff --git a/src/main/java/io/ib67/sfcraft/mixin/MixinMinecraftServer.java b/src/main/java/io/ib67/sfcraft/mixin/MixinMinecraftServer.java index 37c3eb5..6f875d1 100644 --- a/src/main/java/io/ib67/sfcraft/mixin/MixinMinecraftServer.java +++ b/src/main/java/io/ib67/sfcraft/mixin/MixinMinecraftServer.java @@ -1,14 +1,22 @@ package io.ib67.sfcraft.mixin; import io.ib67.sfcraft.Globals; -import io.ib67.sfcraft.IOState; +import io.ib67.sfcraft.JustBackupMod; import net.minecraft.server.MinecraftServer; import org.spongepowered.asm.mixin.Mixin; import org.spongepowered.asm.mixin.Shadow; import org.spongepowered.asm.mixin.injection.At; import org.spongepowered.asm.mixin.injection.Inject; import org.spongepowered.asm.mixin.injection.callback.CallbackInfo; +import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable; +/** + * Skips main-thread saves while a backup holds the write lock. The main thread must never + * block on {@link Globals#WORLD_IO_LOCK}: a save request would join the IOWorkers, which + * are parked on the read lock, wedging the main thread until the backup finishes (and + * tripping the watchdog). The world was saved right before the backup acquired the lock, + * so skipping loses nothing. + */ @Mixin(MinecraftServer.class) public abstract class MixinMinecraftServer { @@ -19,23 +27,21 @@ public abstract class MixinMinecraftServer { protected abstract int computeNextAutosaveInterval(); @Inject(method = "autoSave", at = @At("HEAD"), cancellable = true) - private void backup$checkSaving(CallbackInfo ci) { - // is already backing up - this.ticksUntilAutosave = computeNextAutosaveInterval(); - IOState witness; - while ((witness = Globals.BACKUP_LOCK.compareAndExchange(IOState.IDLE, IOState.SAVING_WORLD)) == IOState.STORAGE_SYNC) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - System.err.println("Error while waiting for chunk sync. " + e); - ci.cancel(); - } + private void backup$skipAutoSaveDuringBackup(CallbackInfo ci) { + if (Globals.WORLD_IO_LOCK.isWriteLocked()) { + // vanilla resets this at the head of autoSave; replicate it on the cancel path + // so the autosave doesn't re-trigger every tick + this.ticksUntilAutosave = computeNextAutosaveInterval(); + JustBackupMod.LOGGER.info("Backup in progress, skipping this auto-save."); + ci.cancel(); } - if (witness == IOState.BACKUP) ci.cancel(); } - @Inject(method = "autoSave", at = @At("RETURN")) - private void backup$disableProtect(CallbackInfo ci) { - Globals.BACKUP_LOCK.setRelease(IOState.IDLE); + @Inject(method = "saveEverything", at = @At("HEAD"), cancellable = true) + private void backup$skipManualSaveDuringBackup(boolean suppressLog, boolean flush, boolean force, CallbackInfoReturnable cir) { + if (Globals.WORLD_IO_LOCK.isWriteLocked()) { + JustBackupMod.LOGGER.warn("Backup in progress, skipping this save request. The world was already saved when the backup started."); + cir.setReturnValue(false); + } } } diff --git a/src/main/java/io/ib67/sfcraft/mixin/MixinRegionBasedStorage.java b/src/main/java/io/ib67/sfcraft/mixin/MixinRegionBasedStorage.java index 81beb7b..45c937d 100644 --- a/src/main/java/io/ib67/sfcraft/mixin/MixinRegionBasedStorage.java +++ b/src/main/java/io/ib67/sfcraft/mixin/MixinRegionBasedStorage.java @@ -1,62 +1,38 @@ package io.ib67.sfcraft.mixin; +import com.llamalad7.mixinextras.injector.wrapmethod.WrapMethod; +import com.llamalad7.mixinextras.injector.wrapoperation.Operation; import io.ib67.sfcraft.Globals; -import io.ib67.sfcraft.IOState; -import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap; -import lombok.SneakyThrows; -import net.minecraft.world.level.chunk.storage.RegionFile; +import net.minecraft.nbt.CompoundTag; +import net.minecraft.world.level.ChunkPos; import net.minecraft.world.level.chunk.storage.RegionFileStorage; -import org.spongepowered.asm.mixin.Final; import org.spongepowered.asm.mixin.Mixin; -import org.spongepowered.asm.mixin.Overwrite; -import org.spongepowered.asm.mixin.Shadow; +/** + * Holds the read side of {@link Globals#WORLD_IO_LOCK} around region file writes and flushes. + * Multiple IOWorker threads (one per dimension and storage kind) may run concurrently under + * the read lock; while the backup worker holds the write lock, all region IO parks here and + * resumes untouched afterwards. + */ @Mixin(RegionFileStorage.class) public abstract class MixinRegionBasedStorage { - @Shadow - @Final - private Long2ObjectLinkedOpenHashMap regionCache; - - /** - * @author iceBear67 - * @reason to wrap the saving process with an exception handler. - */ - @Overwrite - @SneakyThrows - public void flush() { - // allowed states: - // STORAGE_SYNC, AUTO_SAVE, IDLE - // we try - var witness = Globals.BACKUP_LOCK.getAndSet(IOState.STORAGE_SYNC); - if (witness == IOState.BACKUP) { - // yield out and keep blocking. - // The backup worker will wait for us setting this back to backup. - int i =0; - Globals.BACKUP_LOCK.set(IOState.BACKUP); - while (Globals.BACKUP_LOCK.getAcquire() == IOState.BACKUP) { - if(i==0){ - System.out.println("Server is performing backup, ChunkIO will wait for it."); - } - if(i++ > 3000){ - System.err.println("WARNING: BACKUP_LOCK is holding for 10 minutes!"); - System.err.println("ChunkIO will not wait for backup anymore, writting to disk..."); - System.err.println("You may adjust this behaviour in configuration"); - break; - } - Thread.sleep(2000); - } - } - // program point: IDLE, AUTOSAVE, STORAGESYNC + @WrapMethod(method = "write") + private void backup$guardWrite(ChunkPos pos, CompoundTag tag, Operation original) { + long stamp = Globals.WORLD_IO_LOCK.readLock(); try { - for (RegionFile value : this.regionCache.values()) { - value.flush(); - } + original.call(pos, tag); } finally { - // we assert that there are no other threads calling sync() - // so we can just jump back to the witness value. - // witness: AUTO_SAVE or IDLE - Globals.BACKUP_LOCK.compareAndExchange(IOState.STORAGE_SYNC, witness); - Globals.BACKUP_LOCK.setRelease(witness); + Globals.WORLD_IO_LOCK.unlockRead(stamp); + } + } + + @WrapMethod(method = "flush") + private void backup$guardFlush(Operation original) { + long stamp = Globals.WORLD_IO_LOCK.readLock(); + try { + original.call(); + } finally { + Globals.WORLD_IO_LOCK.unlockRead(stamp); } } } diff --git a/src/main/java/io/ib67/sfcraft/strategy/BackupChains.java b/src/main/java/io/ib67/sfcraft/strategy/BackupChains.java new file mode 100644 index 0000000..cd49961 --- /dev/null +++ b/src/main/java/io/ib67/sfcraft/strategy/BackupChains.java @@ -0,0 +1,84 @@ +package io.ib67.sfcraft.strategy; + +import io.ib67.sfcraft.Backup; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; + +/** + * Pure functions over the backup index: resolving restore chains (full base + following + * incrementals) and grouping backups into generations for rotation. Legacy entries + * (created before subject/createdAt existed) are excluded from both — they are restorable + * standalone but never chained or rotated away. + */ +public final class BackupChains { + private BackupChains() { + } + + /** + * Resolves the list of bundles that must be extracted, in order, to restore {@code target}. + * A full backup restores alone; an incremental needs its most recent full base and every + * incremental in between. + * + * @throws IllegalStateException if the target is an incremental without a reachable base + */ + public static List resolveChain(Collection tracked, Backup target) { + if (!target.incremental()) return List.of(target); + if (target.legacy()) { + throw new IllegalStateException("Backup " + target.backupKey() + + " lacks chain metadata (created by an older version) and cannot be restored incrementally."); + } + var chain = new ArrayList(); + tracked.stream() + .filter(b -> !b.legacy() && Objects.equals(target.subject(), b.subject())) + .filter(b -> b.createdAt() < target.createdAt()) + .sorted(Comparator.comparingLong(Backup::createdAt)) + .forEach(b -> { + if (!b.incremental()) chain.clear(); // a newer full base supersedes older history + chain.add(b); + }); + chain.add(target); + if (chain.getFirst().incremental()) { + throw new IllegalStateException("No full backup found before " + target.backupKey() + + "; cannot restore an incremental backup without its full base."); + } + return List.copyOf(chain); + } + + /** + * Groups a subject's backups into generations: each full backup starts a new generation + * that collects the incrementals after it. Incrementals older than the first full backup + * (orphans) belong to no generation. + */ + public static List> generations(Collection tracked, String subject) { + var result = new ArrayList>(); + List current = null; + var sorted = tracked.stream() + .filter(b -> !b.legacy() && Objects.equals(subject, b.subject())) + .sorted(Comparator.comparingLong(Backup::createdAt)) + .toList(); + for (var b : sorted) { + if (!b.incremental()) { + current = new ArrayList<>(); + result.add(current); + } + if (current != null) current.add(b); + } + return result; + } + + /** + * @return the backups that fall out of the newest {@code keepGenerations} generations and + * should be deleted. Whole generations only — an incremental is never separated from its base. + */ + public static List rotationVictims(Collection tracked, String subject, int keepGenerations) { + var generations = generations(tracked, subject); + if (generations.size() <= keepGenerations) return List.of(); + return generations.subList(0, generations.size() - keepGenerations).stream() + .flatMap(List::stream) + .toList(); + } +} diff --git a/src/main/java/io/ib67/sfcraft/strategy/BackupStrategy.java b/src/main/java/io/ib67/sfcraft/strategy/BackupStrategy.java index 205523b..6fef388 100644 --- a/src/main/java/io/ib67/sfcraft/strategy/BackupStrategy.java +++ b/src/main/java/io/ib67/sfcraft/strategy/BackupStrategy.java @@ -6,10 +6,16 @@ import java.nio.file.Path; public interface BackupStrategy { /** - * create a backup - * @return a key to this backup + * Stores the finished bundle. + * + * @param subject name of the backup subject + * @param source directory the bundle was created from + * @param bundle the bundle file to store; the caller deletes it afterwards + * @param incremental whether the bundle only contains changed files + * @param createdAt epoch millis the backup was taken at + * @return metadata describing the stored backup */ - Backup createBackup(String subject, Path source, Path bundle); + Backup createBackup(String subject, Path source, Path bundle, boolean incremental, long createdAt); void recoverBackup(Backup backup, Path restorePath); diff --git a/src/main/java/io/ib67/sfcraft/strategy/BackupTracker.java b/src/main/java/io/ib67/sfcraft/strategy/BackupTracker.java index 7a72816..c92ab58 100644 --- a/src/main/java/io/ib67/sfcraft/strategy/BackupTracker.java +++ b/src/main/java/io/ib67/sfcraft/strategy/BackupTracker.java @@ -1,28 +1,46 @@ package io.ib67.sfcraft.strategy; import io.ib67.sfcraft.Backup; +import io.ib67.sfcraft.Globals; import lombok.Getter; -import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import java.io.IOException; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; import java.nio.file.Path; -import java.util.HashMap; +import java.nio.file.StandardCopyOption; import java.util.Map; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +/** + * Wraps a storage strategy with an in-memory index of known backups, persisted atomically + * to {@code indexPath} after every mutation. The map is concurrent: the backup executor + * writes while the main thread reads it for commands and tab completion. + */ +@Log4j2 public class BackupTracker implements BackupStrategy { protected final BackupStrategy underlying; @Getter - protected final Map trackedBackups; + protected final Map trackedBackups = new ConcurrentHashMap<>(); + protected final Path indexPath; - public BackupTracker(BackupStrategy underlying, Map trackedBackups) { + public BackupTracker(BackupStrategy underlying, Map initial, Path indexPath) { this.underlying = Objects.requireNonNull(underlying); - this.trackedBackups = trackedBackups == null ? new HashMap<>() : trackedBackups; + this.indexPath = Objects.requireNonNull(indexPath); + if (initial != null) { + initial.forEach((k, v) -> { + if (k != null && v != null) trackedBackups.put(k, v); + }); + } } @Override - public Backup createBackup(String subject, Path from, Path pathToBundle) { - var result = underlying.createBackup(subject, from, pathToBundle); + public Backup createBackup(String subject, Path source, Path bundle, boolean incremental, long createdAt) { + var result = underlying.createBackup(subject, source, bundle, incremental, createdAt); trackedBackups.put(result.backupKey(), result); + persist(); return result; } @@ -35,6 +53,26 @@ public class BackupTracker implements BackupStrategy { public void deleteBackup(Backup backup) { underlying.deleteBackup(backup); trackedBackups.remove(backup.backupKey()); + persist(); + } + + /** + * Writes the index to a temp file first and atomically moves it into place, so a crash + * mid-write can never leave a corrupt index behind (which would disable the mod on the + * next startup). + */ + protected synchronized void persist() { + try { + var tmp = indexPath.resolveSibling(indexPath.getFileName() + ".tmp"); + Files.writeString(tmp, Globals.GSON.toJson(trackedBackups)); + try { + Files.move(tmp, indexPath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException e) { + Files.move(tmp, indexPath, StandardCopyOption.REPLACE_EXISTING); + } + } catch (IOException e) { + log.error("Cannot persist backup index to {}", indexPath, e); + } } @Override diff --git a/src/main/java/io/ib67/sfcraft/strategy/LocalBackupStrategy.java b/src/main/java/io/ib67/sfcraft/strategy/LocalBackupStrategy.java index 5e104ee..5585622 100644 --- a/src/main/java/io/ib67/sfcraft/strategy/LocalBackupStrategy.java +++ b/src/main/java/io/ib67/sfcraft/strategy/LocalBackupStrategy.java @@ -25,24 +25,23 @@ public class LocalBackupStrategy implements BackupStrategy { @Override @SneakyThrows - public Backup createBackup(String subject, Path from, Path pathToBundle) { + public Backup createBackup(String subject, Path source, Path bundle, boolean incremental, long createdAt) { var parent = backupParentRoot.resolve(subject); + var fileName = bundle.getFileName().toString(); var result = new Backup( - pathToBundle.getFileName().toString(), - Path.of(subject).resolve(pathToBundle.getFileName()).normalize().toString(), - "local", from.toString(), false, Files.size(pathToBundle)); - if(Files.notExists(parent)) { + fileName, + Path.of(subject).resolve(fileName).normalize().toString(), + "local", source.toString(), subject, incremental, createdAt, Files.size(bundle)); + if (Files.notExists(parent)) { Files.createDirectories(parent); } - //todo new name - Files.move(pathToBundle, parent.resolve(pathToBundle.getFileName())); + Files.move(bundle, parent.resolve(fileName)); return result; } @Override @SneakyThrows public void recoverBackup(Backup backup, Path restorePath) { - // check if it is compressed var backupFile = backupParentRoot.resolve(backup.backupKey()); if (Files.notExists(backupFile)) throw new IllegalArgumentException("Backup " + backupFile + " does not exist"); BundleReader.builder().build().extract(backupFile, restorePath); @@ -51,7 +50,7 @@ public class LocalBackupStrategy implements BackupStrategy { @SneakyThrows @Override public void deleteBackup(Backup backup) { - Validate.isTrue("local".equals(backup.backupKey())); + Validate.isTrue("local".equals(backup.type()), "Not a local backup: %s", backup.backupKey()); var bundlePath = backupParentRoot.resolve(backup.backupKey()); Files.deleteIfExists(bundlePath); } @@ -61,14 +60,14 @@ public class LocalBackupStrategy implements BackupStrategy { public boolean isAvailable() { if (option.diskSizeQuotaBytes() == 0) return true; try (var f = Files.walk(backupParentRoot)) { - if (f.mapToLong(it->{ + if (f.filter(Files::isRegularFile).mapToLong(it -> { try { return Files.size(it); } catch (IOException e) { throw new RuntimeException(e); } }).sum() >= option.diskSizeQuotaBytes()) { - log.warn("Have no space to set another backup! "); + log.warn("Backup disk quota ({} bytes) exceeded, refusing to create new backups!", option.diskSizeQuotaBytes()); return false; } } diff --git a/src/main/java/io/ib67/sfcraft/strategy/S3BackupStrategy.java b/src/main/java/io/ib67/sfcraft/strategy/S3BackupStrategy.java index d84ddae..8de47ad 100644 --- a/src/main/java/io/ib67/sfcraft/strategy/S3BackupStrategy.java +++ b/src/main/java/io/ib67/sfcraft/strategy/S3BackupStrategy.java @@ -7,20 +7,18 @@ import lombok.SneakyThrows; import lombok.extern.log4j.Log4j2; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; -import java.io.IOException; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Objects; -import java.util.concurrent.atomic.AtomicBoolean; @Log4j2 public class S3BackupStrategy implements BackupStrategy { - protected final AtomicBoolean uploading = new AtomicBoolean(false); protected final StorageOption.S3 option; protected final S3Client s3; protected final Path temporaryDownloadPath; @@ -45,45 +43,42 @@ public class S3BackupStrategy implements BackupStrategy { @Override @SneakyThrows - public Backup createBackup(String subject, Path from, Path pathToBundle) { - if (!uploading.compareAndSet(false, true)) { - throw new IllegalStateException("Last upload is not done yet! keep waiting..."); - } - int i = 0; - do { + public Backup createBackup(String subject, Path source, Path bundle, boolean incremental, long createdAt) { + // bundle names embed the creation timestamp, so keys are unique and retries idempotent + var object = option.prefix() + "/" + bundle.getFileName(); + int attempt = 0; + while (true) { try { - var object = option.prefix() + "/" + pathToBundle.getFileName().toString(); s3.putObject(o -> o.bucket(option.bucket()) .contentType("application/octet-stream") - .ifNoneMatch("*") // to avoid content overriding - .key(object).build(), pathToBundle); - return new Backup(pathToBundle.getFileName().toString(), + .key(object), bundle); + return new Backup(bundle.getFileName().toString(), object, "s3", - from.toString(), - false, - Files.size(pathToBundle)); - } catch (IOException e) { - if (i >= option.retryAmount()) { - throw new IllegalStateException("Cannot upload " + pathToBundle + " to S3"); + source.toString(), + subject, incremental, createdAt, + Files.size(bundle)); + } catch (SdkException e) { + if (attempt >= option.retryAmount()) { + throw new IllegalStateException("Cannot upload " + bundle + " to S3 after " + + attempt + " retries", e); } - var nextTry = Math.pow(2, i++); - log.error("Error uploading S3. trying in {}s later. ({}/{})", nextTry, i, option.retryAmount(), e); - Thread.sleep((long) (nextTry * 1000)); - } catch (Exception e) { - throw new IllegalStateException("Unrecoverable exception occurred when uploading " + pathToBundle, e); - } finally { - uploading.set(false); + var backoffSeconds = (long) Math.pow(2, attempt++); + log.error("Error uploading to S3, retrying in {}s ({}/{})", + backoffSeconds, attempt, option.retryAmount(), e); + Thread.sleep(backoffSeconds * 1000); } - } while (true); + } } @Override @SneakyThrows public void recoverBackup(Backup backup, Path restorePath) { - var resp = s3.getObject(b -> b.bucket(option.bucket()).key(backup.backupKey())); var target = temporaryDownloadPath.resolve("s3_" + System.currentTimeMillis()); - try (var fs = Files.newOutputStream(target, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { - resp.transferTo(fs); + try { + try (var resp = s3.getObject(b -> b.bucket(option.bucket()).key(backup.backupKey())); + var fs = Files.newOutputStream(target, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { + resp.transferTo(fs); + } BundleReader.builder().build().extract(target, restorePath); } finally { Files.deleteIfExists(target); @@ -98,10 +93,6 @@ public class S3BackupStrategy implements BackupStrategy { @Override public boolean isAvailable() { - if (uploading.get()) { - log.warn("Last S3 Backup is still uploading!"); - return false; - } return true; } diff --git a/src/test/java/io/ib67/sfcraft/TestBackupChains.java b/src/test/java/io/ib67/sfcraft/TestBackupChains.java new file mode 100644 index 0000000..e3a204c --- /dev/null +++ b/src/test/java/io/ib67/sfcraft/TestBackupChains.java @@ -0,0 +1,88 @@ +package io.ib67.sfcraft; + +import io.ib67.sfcraft.strategy.BackupChains; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestBackupChains { + private static Backup full(String key, String subject, long at) { + return new Backup(key, key, "local", "world", subject, false, at, 1); + } + + private static Backup incr(String key, String subject, long at) { + return new Backup(key, key, "local", "world", subject, true, at, 1); + } + + private static Backup legacy(String key) { + return new Backup(key, key, "local", "world", null, false, 0, 1); + } + + @Test + public void fullBackupRestoresAlone() { + var full = full("f1", "world", 100); + var all = List.of(full, incr("i1", "world", 200)); + assertEquals(List.of(full), BackupChains.resolveChain(all, full)); + } + + @Test + public void incrementalChainsBackToItsBase() { + var f1 = full("f1", "world", 100); + var i1a = incr("i1a", "world", 200); + var i1b = incr("i1b", "world", 300); + var f2 = full("f2", "world", 400); + var i2a = incr("i2a", "world", 500); + var all = List.of(i2a, f1, i1b, f2, i1a); // deliberately unordered + + assertEquals(List.of(f1, i1a, i1b), BackupChains.resolveChain(all, i1b)); + assertEquals(List.of(f2, i2a), BackupChains.resolveChain(all, i2a)); + } + + @Test + public void chainIgnoresOtherSubjects() { + var f1 = full("f1", "world", 100); + var otherFull = full("fo", "nether", 150); + var i1 = incr("i1", "world", 200); + assertEquals(List.of(f1, i1), BackupChains.resolveChain(List.of(f1, otherFull, i1), i1)); + } + + @Test + public void orphanIncrementalIsRejected() { + var i1 = incr("i1", "world", 200); + assertThrows(IllegalStateException.class, + () -> BackupChains.resolveChain(List.of(i1), i1)); + } + + @Test + public void legacyEntryRestoresStandalone() { + var old = legacy("old"); + assertEquals(List.of(old), BackupChains.resolveChain(List.of(old), old)); + } + + @Test + public void rotationKeepsWholeGenerations() { + var f1 = full("f1", "world", 100); + var i1 = incr("i1", "world", 200); + var f2 = full("f2", "world", 300); + var i2 = incr("i2", "world", 400); + var f3 = full("f3", "world", 500); + var all = List.of(f1, i1, f2, i2, f3); + + assertEquals(List.of(), BackupChains.rotationVictims(all, "world", 3)); + assertEquals(List.of(f1, i1), BackupChains.rotationVictims(all, "world", 2)); + assertEquals(List.of(f1, i1, f2, i2), BackupChains.rotationVictims(all, "world", 1)); + } + + @Test + public void rotationNeverTouchesLegacyOrOrphans() { + var orphan = incr("orphan", "world", 50); + var old = legacy("old"); + var f1 = full("f1", "world", 100); + var f2 = full("f2", "world", 200); + var all = List.of(orphan, old, f1, f2); + + assertEquals(List.of(f1), BackupChains.rotationVictims(all, "world", 1)); + } +}