feat: implement backup system with compression and bundling
- Add zstd-jni dependency for compression support - Define Backup record with name, key, incremental, and size fields - Implement BackupManager with backup listing, deletion, and creation methods - Create BackupWorker to handle backup operations with server state coordination - Refactor WorldDir to simplify dimension handling and file listing - Introduce bundling system with SerialBundlerReader^[[1;5P/Writer for efficient storage - Add BackupStrategy interface with LocalBackupStrategy implementation - Support both compressed and uncompressed backup formats - Add config option for parallel backup operations - Remove old backup implementation from JustBackupMod - Update backup file format with metadata footer and magic number validation
This commit is contained in:
1 parent
bc9fab8d22
commit
6a2642ab93
14 files changed
+460
-72
No files matched your search
@@ -0,0 +1,10 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
public record BundlerEntry(
|
||||
String fileName,
|
||||
boolean unGzipped,
|
||||
long fileLength,
|
||||
long offset
|
||||
) {
|
||||
public static final int MAGIC = 0xBACEBACE;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
import java.io.FilterInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
public class EntryInputStream extends FilterInputStream {
|
||||
/**
|
||||
* Creates a {@code FilterInputStream}
|
||||
* by assigning the argument {@code in}
|
||||
* to the field {@code this.in} so as
|
||||
* to remember it for later use.
|
||||
*
|
||||
* @param in the underlying input stream, or {@code null} if
|
||||
* this instance is to be created without an underlying stream.
|
||||
*/
|
||||
protected EntryInputStream(InputStream in) {
|
||||
super(in);
|
||||
}
|
||||
|
||||
public BundlerEntry readEntry() throws IOException {
|
||||
var len = ByteBuffer.wrap(readNBytes(4)).getInt();
|
||||
var gunzipped = len < 0;
|
||||
len = Math.abs(len);
|
||||
var name = new String(readNBytes(len)).intern();
|
||||
var buf = ByteBuffer.wrap(readNBytes(8));
|
||||
var fileLen = buf.getLong();
|
||||
var fileOffset = buf.getLong();
|
||||
return new BundlerEntry(name, gunzipped, fileLen, fileOffset);
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
import java.io.FilterOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
public class EntryOutputStream extends FilterOutputStream {
|
||||
/**
|
||||
* Creates an output stream filter built on top of the specified
|
||||
* underlying output stream.
|
||||
*
|
||||
* @param out the underlying output stream to be assigned to
|
||||
* the field {@code this.out} for later use, or
|
||||
* {@code null} if this instance is to be
|
||||
* created without an underlying stream.
|
||||
*/
|
||||
public EntryOutputStream(OutputStream out) {
|
||||
super(out);
|
||||
}
|
||||
|
||||
public void writeMagic() throws IOException {
|
||||
out.write(BundlerEntry.MAGIC);
|
||||
}
|
||||
|
||||
public void write(BundlerEntry entry) throws IOException {
|
||||
var fileName = entry.fileName().getBytes(StandardCharsets.UTF_8);
|
||||
var buf = ByteBuffer.allocate(4).putInt(entry.unGzipped() ? fileName.length * -1 : fileName.length);
|
||||
out.write(buf.array());
|
||||
out.write(fileName);
|
||||
buf = ByteBuffer.allocate(8 + 8);
|
||||
buf.putLong(entry.fileLength());
|
||||
buf.putLong(entry.offset());
|
||||
out.write(buf.array());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
import com.github.luben.zstd.Zstd;
|
||||
import com.github.luben.zstd.ZstdInputStreamNoFinalizer;
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.Channels;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Objects;
|
||||
|
||||
public class SerialBundlerReader implements Closeable {
|
||||
protected final InputStream in;
|
||||
protected final FileChannel inChannel; // for direct copy
|
||||
|
||||
protected final EntryInputStream metadataIn;
|
||||
protected final byte[] buffer = new byte[4096];
|
||||
protected boolean entryRead;
|
||||
|
||||
public SerialBundlerReader(InputStream in, InputStream metadataIn) {
|
||||
this.in = Objects.requireNonNull(in);
|
||||
this.metadataIn = new EntryInputStream(Objects.requireNonNull(metadataIn));
|
||||
inChannel = null;
|
||||
}
|
||||
|
||||
public SerialBundlerReader(Path path) throws IOException {
|
||||
var raf = new RandomAccessFile(path.toFile(), "r");
|
||||
var totalLen = Files.size(path);
|
||||
raf.seek(totalLen - 4);
|
||||
var metadataLength = raf.readInt();
|
||||
var metadataBegin = totalLen - 4 - metadataLength;
|
||||
var metadataBytes = new byte[metadataLength];
|
||||
raf.seek(metadataBegin);
|
||||
raf.readFully(metadataBytes);
|
||||
|
||||
this.metadataIn = new EntryInputStream(new ZstdInputStreamNoFinalizer(new ByteArrayInputStream(metadataBytes)));
|
||||
if (ByteBuffer.wrap(metadataIn.readNBytes(4)).getInt() != BundlerEntry.MAGIC) {
|
||||
throw new IllegalArgumentException("Invalid Bundler entry");
|
||||
}
|
||||
// check compressed
|
||||
raf.seek(0);
|
||||
var fis = Channels.newInputStream(raf.getChannel());
|
||||
if (raf.readInt() == Zstd.magicNumber()) {
|
||||
in = new ZstdInputStreamNoFinalizer(fis);
|
||||
inChannel = null;
|
||||
} else {
|
||||
in = fis;
|
||||
inChannel = raf.getChannel();
|
||||
}
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
public boolean hasEntry() {
|
||||
return metadataIn.available() > 0;
|
||||
}
|
||||
|
||||
public BundlerEntry readEntry() throws IOException {
|
||||
var entry = metadataIn.readEntry();
|
||||
entryRead = true;
|
||||
return entry;
|
||||
}
|
||||
|
||||
public void writeTo(FileChannel channel, BundlerEntry entry) throws IOException {
|
||||
if (inChannel != null) {
|
||||
inChannel.transferTo(entry.offset(), entry.fileLength(), channel);
|
||||
} else {
|
||||
writeTo(Channels.newOutputStream(channel), entry);
|
||||
}
|
||||
}
|
||||
|
||||
public void writeTo(OutputStream out, BundlerEntry entry) throws IOException {
|
||||
Objects.requireNonNull(entry);
|
||||
var remaining = entry.fileLength();
|
||||
var buffer = this.buffer;
|
||||
var totalRead = 0;
|
||||
do {
|
||||
var read = in.read(buffer, 0, (int) Math.min(remaining - totalRead, buffer.length));
|
||||
out.write(buffer, 0, read);
|
||||
totalRead += read;
|
||||
} while (totalRead < remaining);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
in.close();
|
||||
metadataIn.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.Channels;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
@AllArgsConstructor
|
||||
public class SerialBundlerWriter implements Closeable {
|
||||
protected final FileChannel channel;
|
||||
protected final OutputStream out;
|
||||
protected long offset;
|
||||
@Getter
|
||||
protected List<BundlerEntry> entries;
|
||||
|
||||
public SerialBundlerWriter(OutputStream out) {
|
||||
Objects.requireNonNull(out);
|
||||
this.out = out;
|
||||
channel = null;
|
||||
}
|
||||
|
||||
public SerialBundlerWriter(FileChannel channel, long offset) {
|
||||
Objects.requireNonNull(channel);
|
||||
this.offset = offset;
|
||||
this.channel = channel;
|
||||
out = null;
|
||||
}
|
||||
|
||||
public void writeGunzip(Path path, String fileName) throws IOException {
|
||||
try (var gzip = new GZIPInputStream(Files.newInputStream(path, StandardOpenOption.READ))) {
|
||||
var len = channel != null ? gzip.transferTo(Channels.newOutputStream(channel)) :
|
||||
gzip.transferTo(out);
|
||||
entries.add(new BundlerEntry(fileName, true, len, offset));
|
||||
offset += len;
|
||||
}
|
||||
}
|
||||
|
||||
public void writePlain(Path path, String fileName) throws IOException {
|
||||
if (channel != null) {
|
||||
writeZC(path, fileName);
|
||||
return;
|
||||
}
|
||||
try (var in = Files.newInputStream(path)) {
|
||||
in.transferTo(out);
|
||||
}
|
||||
}
|
||||
|
||||
private void writeZC(Path path, String fileName) throws IOException {
|
||||
try (var in = FileChannel.open(path, StandardOpenOption.READ)) {
|
||||
var begin = offset;
|
||||
var currentPosition = offset;
|
||||
long transferred = 0;
|
||||
while ((transferred = in.transferTo(currentPosition, in.size() - transferred, channel)) != 0) {
|
||||
currentPosition += transferred;
|
||||
}
|
||||
offset = currentPosition;
|
||||
entries.add(new BundlerEntry(
|
||||
fileName, false, offset - begin, begin
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
if (out != null) out.close();
|
||||
if (channel != null) channel.close();
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user