fix: record delete operations
This commit is contained in:
13 files changed
+240
-37
No files matched your search
@@ -16,5 +16,7 @@ public record BundleEntry(
|
||||
) {
|
||||
public static short ATTR_REASSEMBLE = 1;
|
||||
public static short ATTR_SYMLINK = 2;
|
||||
/** Tombstone: the file was deleted since the base backup; the reader removes it on restore. */
|
||||
public static short ATTR_DELETE = 4;
|
||||
public static short MAGIC = (short) 0xA2E3;
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import java.io.EOFException;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.file.DirectoryNotEmptyException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
@@ -21,15 +22,17 @@ public class BundleReader {
|
||||
protected final boolean linkSymbol;
|
||||
private final byte[] buffer = new byte[4096];
|
||||
protected final boolean skipOldEntry;
|
||||
protected final boolean ignoreDeletions;
|
||||
protected final ByteBufAllocator allocator;
|
||||
protected final byte[] dict;
|
||||
|
||||
@Builder
|
||||
public BundleReader(long maxTime, long minTime, boolean linkSymbol, boolean skipOldEntry, ByteBufAllocator allocator, byte[] dict) {
|
||||
public BundleReader(long maxTime, long minTime, boolean linkSymbol, boolean skipOldEntry, boolean ignoreDeletions, ByteBufAllocator allocator, byte[] dict) {
|
||||
this.maxTime = maxTime == 0 ? Long.MAX_VALUE : maxTime;
|
||||
this.minTime = minTime;
|
||||
this.linkSymbol = linkSymbol;
|
||||
this.skipOldEntry = skipOldEntry;
|
||||
this.ignoreDeletions = ignoreDeletions;
|
||||
this.allocator = allocator == null ? ByteBufAllocator.DEFAULT : allocator;
|
||||
this.dict = dict;
|
||||
}
|
||||
@@ -62,6 +65,22 @@ public class BundleReader {
|
||||
readMaps.put(entry.name(), time);
|
||||
|
||||
var path = resolveSafely(destRoot, entry.name());
|
||||
if ((entry.attribute() & BundleEntry.ATTR_DELETE) != 0) {
|
||||
// tombstone from an incremental: mirror the deletion instead of leaving a
|
||||
// resurrected file from the base backup. Deleting a path not present in the
|
||||
// base is a harmless no-op. When ignoreDeletions is set the tombstone is
|
||||
// skipped, so files deleted after the base are kept on restore.
|
||||
if (!ignoreDeletions) {
|
||||
try {
|
||||
Files.deleteIfExists(path);
|
||||
} catch (DirectoryNotEmptyException e) {
|
||||
// a directory tombstone; its contained files are removed by their own
|
||||
// tombstones and an empty leftover directory is harmless
|
||||
}
|
||||
}
|
||||
skipExactly(in, entry.length());
|
||||
continue;
|
||||
}
|
||||
var parentDir = path.getParent();
|
||||
if (Files.notExists(parentDir)) Files.createDirectories(parentDir);
|
||||
if ((entry.attribute() & BundleEntry.ATTR_SYMLINK) != 0) {
|
||||
|
||||
@@ -133,6 +133,20 @@ public class BundleWriter implements Closeable {
|
||||
Path pathToBundle,
|
||||
Iterable<Path> path,
|
||||
UnaryOperator<BundleWriterBuilder> config
|
||||
) throws IOException {
|
||||
createBundle(pathToBundle, path, List.of(), config);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param deletedRelPaths bundle-relative paths of files deleted since the base backup;
|
||||
* written as tombstones so a chained restore mirrors deletions
|
||||
* instead of resurrecting files. Empty for full backups.
|
||||
*/
|
||||
public static void createBundle(
|
||||
Path pathToBundle,
|
||||
Iterable<Path> path,
|
||||
Iterable<String> deletedRelPaths,
|
||||
UnaryOperator<BundleWriterBuilder> config
|
||||
) throws IOException {
|
||||
try (var channel = FileChannel.open(pathToBundle,
|
||||
StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) {
|
||||
@@ -144,6 +158,9 @@ public class BundleWriter implements Closeable {
|
||||
var s = p.toString();
|
||||
writer.write(p, (s.contains("region") && s.endsWith(".mca")) || s.endsWith(".dat"));
|
||||
}
|
||||
for (var rel : deletedRelPaths) {
|
||||
writer.writeTombstone(rel);
|
||||
}
|
||||
}
|
||||
// the zstd epilogue is flushed by writer.close(); fsync before closing so a crash
|
||||
// right after "backup success" cannot leave a hollow bundle behind
|
||||
@@ -185,6 +202,14 @@ public class BundleWriter implements Closeable {
|
||||
writePlain(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Records that {@code relPath} was deleted since the base backup. Carries no payload; on
|
||||
* restore the reader removes the file so a chained restore mirrors the deletion.
|
||||
*/
|
||||
public void writeTombstone(String relPath) throws IOException {
|
||||
outputStream.writeEntryHeader(relPath, BundleEntry.ATTR_DELETE, 0, System.currentTimeMillis());
|
||||
}
|
||||
|
||||
private void writeSymbol(Path path) throws IOException {
|
||||
if (!Files.isSymbolicLink(path)) return;
|
||||
var target = Files.readSymbolicLink(path);
|
||||
|
||||
@@ -41,6 +41,80 @@ public class TestBundleReader {
|
||||
assertEquals("untouched", Files.readString(dest.resolve("b.txt")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTombstoneRemovesFileOnRestore(@TempDir Path tmp) throws IOException {
|
||||
var src = Files.createDirectories(tmp.resolve("src"));
|
||||
Files.writeString(src.resolve("keep.txt"), "keep");
|
||||
Files.writeString(src.resolve("gone.txt"), "gone");
|
||||
var full = tmp.resolve("full.swb.zst");
|
||||
BundleWriter.createBundle(full, List.of(src.resolve("keep.txt"), src.resolve("gone.txt")),
|
||||
c -> c.relativeRoot(src));
|
||||
|
||||
// gone.txt was deleted; the incremental carries only a tombstone for it
|
||||
var incr = tmp.resolve("incr.swb.zst");
|
||||
BundleWriter.createBundle(incr, List.of(), List.of("gone.txt"), c -> c.relativeRoot(src));
|
||||
|
||||
var dest = tmp.resolve("dest");
|
||||
var reader = BundleReader.builder().build();
|
||||
reader.extract(full, dest);
|
||||
assertTrue(Files.exists(dest.resolve("gone.txt")), "base restore should lay down the file first");
|
||||
|
||||
reader.extract(incr, dest);
|
||||
assertTrue(Files.exists(dest.resolve("keep.txt")), "unrelated files must survive the tombstone");
|
||||
assertFalse(Files.exists(dest.resolve("gone.txt")),
|
||||
"a tombstoned file must be removed on chained restore, not resurrected");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIgnoreDeletionsKeepsTombstonedFile(@TempDir Path tmp) throws IOException {
|
||||
var src = Files.createDirectories(tmp.resolve("src"));
|
||||
Files.writeString(src.resolve("keep.txt"), "keep");
|
||||
Files.writeString(src.resolve("gone.txt"), "gone");
|
||||
var full = tmp.resolve("full.swb.zst");
|
||||
BundleWriter.createBundle(full, List.of(src.resolve("keep.txt"), src.resolve("gone.txt")),
|
||||
c -> c.relativeRoot(src));
|
||||
var incr = tmp.resolve("incr.swb.zst");
|
||||
BundleWriter.createBundle(incr, List.of(), List.of("gone.txt"), c -> c.relativeRoot(src));
|
||||
|
||||
var dest = tmp.resolve("dest");
|
||||
var reader = BundleReader.builder().ignoreDeletions(true).build();
|
||||
reader.extract(full, dest);
|
||||
reader.extract(incr, dest);
|
||||
|
||||
assertTrue(Files.exists(dest.resolve("gone.txt")),
|
||||
"with ignoreDeletions the tombstone is skipped and the file is kept");
|
||||
assertEquals("gone", Files.readString(dest.resolve("gone.txt")));
|
||||
assertEquals("keep", Files.readString(dest.resolve("keep.txt")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTombstoneForUnknownPathIsHarmless(@TempDir Path tmp) throws IOException {
|
||||
var src = Files.createDirectories(tmp.resolve("src"));
|
||||
Files.writeString(src.resolve("a.txt"), "a");
|
||||
var bundle = tmp.resolve("b.swb.zst");
|
||||
// tombstone a nested path that was never in the base backup
|
||||
BundleWriter.createBundle(bundle, List.of(src.resolve("a.txt")),
|
||||
List.of("region/r.0.0.mca"), c -> c.relativeRoot(src));
|
||||
|
||||
var dest = tmp.resolve("dest");
|
||||
BundleReader.builder().build().extract(bundle, dest);
|
||||
assertEquals("a", Files.readString(dest.resolve("a.txt")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTombstoneCannotEscapeDestination(@TempDir Path tmp) throws IOException {
|
||||
var outside = Files.writeString(tmp.resolve("outside.txt"), "precious");
|
||||
var bundle = tmp.resolve("evil.swb.zst");
|
||||
try (var zstd = new ZstdOutputStreamNoFinalizer(Files.newOutputStream(bundle), 3);
|
||||
var out = new EntryOutputStream(zstd)) {
|
||||
out.writeEntryHeader("../outside.txt", io.ib67.sfcraft.bundler.BundleEntry.ATTR_DELETE, 0,
|
||||
System.currentTimeMillis());
|
||||
}
|
||||
var dest = tmp.resolve("dest");
|
||||
assertThrows(IOException.class, () -> BundleReader.builder().build().extract(bundle, dest));
|
||||
assertTrue(Files.exists(outside), "a tombstone must not delete files outside the destination");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testZipSlipIsRejected(@TempDir Path tmp) throws IOException {
|
||||
var bundle = tmp.resolve("evil.swb.zst");
|
||||
|
||||
Reference in new issue
Block a user