refactor: Restructure project into multi-module setup and improve bundler
- Convert project to multi-module structure with 'bundler' subproject - Extract bundler functionality into separate module with proper dependencies - Improve IO state management with atomic reference for better synchronization - Enhance bundler performance with parallel processing support - Add CLI interface to bundler with various compression options - Fix file handling issues in bundle reader/writer - Update compression configuration and remove redundant options - Improve region file synchronization with proper state management - Add dictionary support for better compression ratios - Refactor backup strategies to use new bundler API - Update dependencies and build configuration - Fix path resolution issues in bundle operations `
This commit is contained in:
1 parent
3a4256cee1
commit
28280bc162
26 files changed
+576
-507
No files matched your search
+20
-19
@@ -5,6 +5,9 @@ plugins {
|
||||
id("com.gradleup.shadow") version '9.1.0'
|
||||
}
|
||||
|
||||
allprojects {
|
||||
apply plugin: "io.freefair.lombok"
|
||||
apply plugin: 'java'
|
||||
version = project.mod_version
|
||||
group = project.maven_group
|
||||
|
||||
@@ -22,6 +25,9 @@ repositories {
|
||||
name = "Kalculos"
|
||||
url = "https://repo.sfclub.cc/snapshots"
|
||||
}
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
loom {
|
||||
@@ -34,6 +40,19 @@ loom {
|
||||
}
|
||||
}
|
||||
|
||||
java {
|
||||
// Loom will automatically attach sourcesJar to a RemapSourcesJar task and to the "build" task
|
||||
// if it is present.
|
||||
// If you remove this line, sources will not be generated.
|
||||
withSourcesJar()
|
||||
|
||||
sourceCompatibility = JavaVersion.VERSION_21
|
||||
targetCompatibility = JavaVersion.VERSION_21
|
||||
}
|
||||
tasks.withType(JavaCompile).configureEach {
|
||||
it.options.release = 21
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
dependencies {
|
||||
@@ -45,8 +64,8 @@ dependencies {
|
||||
// Fabric API. This is technically optional, but you probably want it anyway.
|
||||
modImplementation "net.fabricmc.fabric-api:fabric-api:${project.fabric_version}"
|
||||
// https://mvnrepository.com/artifact/software.amazon.awssdk/s3
|
||||
api project(":bundler")
|
||||
shadow 'software.amazon.awssdk:s3:2.34.0'
|
||||
shadow("io.ib67.kiwi:lang:1.2.1-SNAPSHOT")
|
||||
shadow 'com.github.luben:zstd-jni:1.5.7-4'
|
||||
}
|
||||
|
||||
@@ -64,26 +83,8 @@ processResources {
|
||||
expand "version": inputs.properties.version
|
||||
}
|
||||
}
|
||||
|
||||
tasks.withType(JavaCompile).configureEach {
|
||||
it.options.release = 21
|
||||
}
|
||||
|
||||
java {
|
||||
// Loom will automatically attach sourcesJar to a RemapSourcesJar task and to the "build" task
|
||||
// if it is present.
|
||||
// If you remove this line, sources will not be generated.
|
||||
withSourcesJar()
|
||||
|
||||
sourceCompatibility = JavaVersion.VERSION_21
|
||||
targetCompatibility = JavaVersion.VERSION_21
|
||||
}
|
||||
|
||||
jar {
|
||||
inputs.property "archivesName", project.base.archivesName
|
||||
manifest {
|
||||
attributes(Map.of("Main-Class", "io.ib67.sfcraft.bundler.Main"))
|
||||
}
|
||||
from("LICENSE") {
|
||||
rename { "${it}_${inputs.properties.archivesName}" }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
plugins {
|
||||
id 'java'
|
||||
id 'java-library'
|
||||
id("io.freefair.lombok") version "9.0.0-rc2"
|
||||
id("com.gradleup.shadow") version '9.1.0'
|
||||
}
|
||||
|
||||
|
||||
dependencies {
|
||||
api 'com.github.luben:zstd-jni:1.5.7-4'
|
||||
compileOnly 'io.netty:netty-all:4.2.6.Final'
|
||||
api "io.ib67.kiwi:lang:1.2.3-SNAPSHOT"
|
||||
api 'io.netty:netty-all:4.2.6.Final'
|
||||
}
|
||||
|
||||
|
||||
jar {
|
||||
manifest {
|
||||
attributes(Map.of("Main-Class", "io.ib67.sfcraft.bundler.Main"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
public record BundleEntry(
|
||||
short attribute,
|
||||
long fileLen,
|
||||
String name
|
||||
) {
|
||||
public static short ATTR_GUNZIP = 1;
|
||||
public static short MAGIC = (short) 0xA2E3;
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
import com.github.luben.zstd.Zstd;
|
||||
import com.github.luben.zstd.ZstdInputStreamNoFinalizer;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.nio.channels.Channels;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
|
||||
public class BundleReader {
|
||||
public static void extract(Path bundlePath, Path destination) throws IOException {
|
||||
var buffer = new byte[4096];
|
||||
try (var in = new EntryInputStream(
|
||||
new ZstdInputStreamNoFinalizer(Files.newInputStream(bundlePath)))) {
|
||||
while (in.available() > 0) {
|
||||
var entry = in.readEntry();
|
||||
var len = entry.fileLen();
|
||||
var dstPath = destination.resolve(Path.of(entry.name()));
|
||||
if (Files.notExists(dstPath.getParent())) {
|
||||
Files.createDirectories(dstPath.getParent());
|
||||
}
|
||||
try (var out = Files.newOutputStream(dstPath,
|
||||
StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
|
||||
var read = 0;
|
||||
var toRead = len;
|
||||
while (toRead > 0) {
|
||||
read = in.read(buffer, 0, (int) Math.min(buffer.length, toRead));
|
||||
toRead -= read;
|
||||
out.write(buffer, 0, read);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
import com.github.luben.zstd.ZstdDictCompress;
|
||||
import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
|
||||
import io.ib67.kiwi.routine.Uni;
|
||||
import io.netty.buffer.AdaptiveByteBufAllocator;
|
||||
import io.netty.buffer.ByteBufAllocator;
|
||||
import lombok.Builder;
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.function.UnaryOperator;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
public class BundleWriter implements Closeable {
|
||||
public static Comparator<Path> SORT = BundleWriter::preferRegion;
|
||||
protected static final ByteBufAllocator ALLOC = new AdaptiveByteBufAllocator();
|
||||
protected final boolean allowGunzip;
|
||||
protected final int compressionLevel;
|
||||
protected final Path relativeRoot;
|
||||
protected final EntryOutputStream outputStream;
|
||||
protected final byte[] buffer;
|
||||
|
||||
@SneakyThrows
|
||||
@Builder
|
||||
public BundleWriter(
|
||||
boolean allowGunzip,
|
||||
int compressionLevel,
|
||||
int maxWorkers,
|
||||
Path relativeRoot,
|
||||
OutputStream outputStream,
|
||||
byte[] dictionary
|
||||
) {
|
||||
this.allowGunzip = allowGunzip;
|
||||
this.compressionLevel = compressionLevel;
|
||||
this.relativeRoot = relativeRoot;;
|
||||
var zstdOut = new ZstdOutputStreamNoFinalizer(outputStream, compressionLevel);
|
||||
if (dictionary != null) {
|
||||
zstdOut.setDict(new ZstdDictCompress(dictionary, compressionLevel));
|
||||
}
|
||||
if(maxWorkers > 1) zstdOut.setWorkers(maxWorkers);
|
||||
this.outputStream = new EntryOutputStream(zstdOut);
|
||||
this.buffer = new byte[4096];
|
||||
}
|
||||
|
||||
public static CompletableFuture<List<Path>> createBundleParallelized(
|
||||
Path parentOfBundle,
|
||||
List<Path> _paths,
|
||||
int threshold,
|
||||
Executor executor,
|
||||
UnaryOperator<BundleWriterBuilder> config
|
||||
) throws IOException {
|
||||
var paths = new ArrayList<>(_paths);
|
||||
if (Files.notExists(parentOfBundle)) {
|
||||
Files.createDirectories(parentOfBundle);
|
||||
}
|
||||
// Shuffle once to make elements distribute evenly, then split and sort for
|
||||
// the final sequence
|
||||
Collections.shuffle(paths);
|
||||
if (paths.size() >= threshold) {
|
||||
var parallelism = Math.min(
|
||||
Runtime.getRuntime().availableProcessors() / 2
|
||||
, paths.size() / threshold
|
||||
); //todo tuning and debug
|
||||
|
||||
var parted = partition(paths, paths.size() / parallelism);
|
||||
var futures = new ArrayList<CompletableFuture<Path>>();
|
||||
for (int i = 0; i < parted.size(); i++) {
|
||||
var pathList = parted.get(i);
|
||||
var output = parentOfBundle.resolve("Worker_" + System.currentTimeMillis() + "_" + i + ".jpack");
|
||||
var future = CompletableFuture.supplyAsync(() -> createBundleWorker(pathList, output, config), executor);
|
||||
futures.add(future);
|
||||
}
|
||||
return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).thenApply((v) ->
|
||||
Uni.from(futures::forEach).map(CompletableFuture::join).toList());
|
||||
} else {
|
||||
return CompletableFuture.supplyAsync(() -> {
|
||||
var singleResult = parentOfBundle.resolve("result.zst");
|
||||
try {
|
||||
createBundle(singleResult, paths, config);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return List.of(singleResult);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
private static Path createBundleWorker(
|
||||
List<Path> task,
|
||||
Path output,
|
||||
UnaryOperator<BundleWriterBuilder> config
|
||||
) {
|
||||
task.sort(SORT);
|
||||
createBundle(output, task, config);
|
||||
return output;
|
||||
}
|
||||
|
||||
private static <E> List<List<E>> partition(List<E> toPartition, int eachSize) {
|
||||
var result = new ArrayList<List<E>>();
|
||||
var remain = toPartition.size() % eachSize;
|
||||
var groups = (toPartition.size() - remain) / eachSize;
|
||||
for (int i = 0; i < groups; i++) {
|
||||
result.add(toPartition.subList(i * eachSize, (i + 1) * eachSize));
|
||||
}
|
||||
if (remain != 0) result.add(toPartition.subList(groups * eachSize, groups * eachSize + remain));
|
||||
return result;
|
||||
}
|
||||
|
||||
public static int preferRegion(Path a, Path b) {
|
||||
boolean aMatch = a.getParent().toString().equals("region") && a.endsWith(".mca");
|
||||
boolean bMatch = b.getParent().toString().equals("region") && b.endsWith(".mca");
|
||||
|
||||
if (aMatch && !bMatch) return -1;
|
||||
if (!aMatch && bMatch) return 1;
|
||||
return a.compareTo(b);
|
||||
}
|
||||
|
||||
public static void createBundle(
|
||||
Path pathToBundle,
|
||||
Iterable<Path> path,
|
||||
UnaryOperator<BundleWriterBuilder> config
|
||||
) throws IOException {
|
||||
try (var writer = config.apply(BundleWriter.builder())
|
||||
.outputStream(Files.newOutputStream(pathToBundle))
|
||||
.build()) {
|
||||
for (var p : path) {
|
||||
if (!Files.isRegularFile(p) || Files.size(p) == 0) continue;
|
||||
var s = p.toString();
|
||||
writer.write(p, (s.contains("region") && s.endsWith(".mca")) || s.endsWith(".dat"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void write(Path path, boolean suggestGunzip) throws IOException {
|
||||
var size = Files.size(path);
|
||||
if (suggestGunzip) {
|
||||
boolean isGzip;
|
||||
try (var raf = new RandomAccessFile(path.toFile(), "r")) {
|
||||
isGzip = raf.read() == 0x8b && raf.read() == 0x1f;
|
||||
}
|
||||
if (isGzip) {
|
||||
var buffer = ALLOC.buffer((int) size * 2);
|
||||
try (var gunzip = new GZIPInputStream(new FileInputStream(path.toFile()))) {
|
||||
while (buffer.writeBytes(gunzip, 4096) != -1) ;
|
||||
outputStream.writeEntryHeader( // bug
|
||||
relativeRoot.relativize(path).toString(),
|
||||
BundleEntry.ATTR_GUNZIP,
|
||||
size
|
||||
);
|
||||
buffer.readBytes(outputStream, buffer.readableBytes());
|
||||
} catch (IOException e) {
|
||||
writePlain(path);
|
||||
} finally {
|
||||
buffer.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
writePlain(path);
|
||||
}
|
||||
|
||||
private void writePlain(Path path) throws IOException {
|
||||
var rPath = relativeRoot.toAbsolutePath().relativize(path.toAbsolutePath());
|
||||
var buffer = this.buffer;
|
||||
var outputStream = this.outputStream;
|
||||
outputStream.writeEntryHeader(rPath.toString(), (short) 0, Files.size(path));
|
||||
var read = 0;
|
||||
try (var fs = Files.newInputStream(path)) {
|
||||
while ((read = fs.read(buffer)) > 0) {
|
||||
outputStream.write(buffer, 0, read);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
outputStream.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
import io.netty.buffer.ByteBuf;
|
||||
import io.netty.buffer.Unpooled;
|
||||
|
||||
import java.io.FilterInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
public class EntryInputStream extends FilterInputStream {
|
||||
private final ByteBuffer buffer;
|
||||
|
||||
/**
|
||||
* Creates a {@code FilterInputStream}
|
||||
* by assigning the argument {@code in}
|
||||
* to the field {@code this.in}
|
||||
* to remember it for later use.
|
||||
*
|
||||
* @param in the underlying input stream, or {@code null} if
|
||||
* this instance is to be created without an underlying stream.
|
||||
*/
|
||||
protected EntryInputStream(InputStream in) {
|
||||
super(in);
|
||||
this.buffer = ByteBuffer.allocate(4 + 8 + 4);
|
||||
}
|
||||
|
||||
public BundleEntry readEntry() throws IOException {
|
||||
buffer.clear();
|
||||
in.read(buffer.array(), 0, 4+8+4);
|
||||
var magic = buffer.getShort();
|
||||
if (magic != BundleEntry.MAGIC) throw new IOException("Invalid magic number");
|
||||
var attribute = buffer.getShort();
|
||||
var fileLen = buffer.getLong();
|
||||
var lenName = buffer.getInt();
|
||||
var name = new String(in.readNBytes(lenName)).intern();
|
||||
return new BundleEntry(attribute, fileLen, name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
super.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
import io.netty.buffer.ByteBufAllocator;
|
||||
import io.netty.buffer.Unpooled;
|
||||
|
||||
import java.io.FilterOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
public class EntryOutputStream extends FilterOutputStream {
|
||||
protected final ByteBuffer buffer;
|
||||
/**
|
||||
* Creates an output stream filter built on top of the specified
|
||||
* underlying output stream.
|
||||
*
|
||||
* @param out the underlying output stream to be assigned to
|
||||
* the field {@code this.out} for later use, or
|
||||
* {@code null} if this instance is to be
|
||||
* created without an underlying stream.
|
||||
*/
|
||||
public EntryOutputStream(OutputStream out) {
|
||||
super(out);
|
||||
this.buffer = ByteBuffer.allocate(4+8+4);
|
||||
}
|
||||
|
||||
public void writeEntryHeader(String path, short attribute, long fileLen) throws IOException {
|
||||
var name = path.getBytes(StandardCharsets.UTF_8);
|
||||
buffer.clear();
|
||||
buffer.putShort(BundleEntry.MAGIC);
|
||||
buffer.putShort(attribute);
|
||||
buffer.putLong(fileLen);
|
||||
buffer.putInt(name.length);
|
||||
out.write(buffer.array());
|
||||
out.write(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(byte[] b, int off, int len) throws IOException {
|
||||
out.write(b, off, len);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(byte[] b) throws IOException {
|
||||
out.write(b);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
import com.github.luben.zstd.Zstd;
|
||||
import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
|
||||
import io.ib67.kiwi.ArgOpts;
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ForkJoinPool;
|
||||
import java.util.function.UnaryOperator;
|
||||
|
||||
public class Main {
|
||||
@SneakyThrows
|
||||
public static void main(String[] args) {
|
||||
var opts = ArgOpts.builder()
|
||||
.args(args).description("Tools for .jpack format bundle").programName("jpack").build();
|
||||
var in = opts.string("in", "Input. Can be directory or jpack bundle", null);
|
||||
var out = opts.string("out", "Output. Can be jpack bundle or directory", null);
|
||||
var help = opts.bool("help", false);
|
||||
var allowGunzip = opts.bool("gunzip",
|
||||
"Should we recompress NBT files for maximum compression rate (depends on compress)", true);
|
||||
var parallel = opts.bool("parallel", "Use parallelized bundler. This conflicts with compress options", false);
|
||||
var parallelThreshold = opts.integer("threshold", "The threshold of parallel bundling", 512);
|
||||
var parallelCompression = opts.integer("parallel-compression", "The worker threads used for parallel compression. Do not confuse this with --parallel, which focus on bundling.\n" +
|
||||
"This option will be ignored when --parallel is present.", 1);
|
||||
var compressionLevel = opts.integer("level", "Zstd compression level", Zstd.defaultCompressionLevel());
|
||||
var dictionaryPath = opts.string("dict", "Zstd dictionary file", null);
|
||||
if (help) {
|
||||
opts.printHelp(System.out);
|
||||
return;
|
||||
}
|
||||
var dict = (dictionaryPath != null) ? Files.readAllBytes(Path.of(dictionaryPath)) : null;
|
||||
if (in == null || out == null) {
|
||||
System.out.println("Both in and out should be provided");
|
||||
return;
|
||||
}
|
||||
var inPath = Path.of(in);
|
||||
var outPath = Path.of(out);
|
||||
if (!Files.exists(inPath)) {
|
||||
System.out.println("The input isn't exists.");
|
||||
return;
|
||||
}
|
||||
if (Files.exists(outPath)) {
|
||||
System.out.println("The output is already exists. Please move it to other places");
|
||||
return;
|
||||
}
|
||||
if (in.endsWith(".jpack")) {
|
||||
BundleReader.extract(inPath, outPath);
|
||||
} else if (out.endsWith(".jpack")) {
|
||||
if (!Files.isDirectory(inPath))
|
||||
throw new IllegalArgumentException("Your input should be a world directory.");
|
||||
UnaryOperator<BundleWriter.BundleWriterBuilder> cfg = it -> it
|
||||
.allowGunzip(allowGunzip)
|
||||
.relativeRoot(outPath.toAbsolutePath().getParent())
|
||||
.dictionary(dict)
|
||||
.compressionLevel(compressionLevel);
|
||||
List<Path> paths;
|
||||
try (var _paths = Files.walk(inPath)) {
|
||||
paths = _paths.toList();
|
||||
}
|
||||
if (parallel) {
|
||||
try (var fj = ForkJoinPool.commonPool()) {
|
||||
BundleWriter.createBundleParallelized(
|
||||
outPath.getParent(),
|
||||
paths,
|
||||
parallelThreshold, fj,
|
||||
b -> cfg.apply(b).maxWorkers(0)
|
||||
).join();
|
||||
}
|
||||
} else {
|
||||
BundleWriter.createBundle(
|
||||
outPath,
|
||||
paths,
|
||||
c -> cfg.apply(c).maxWorkers(parallelCompression)
|
||||
);
|
||||
}
|
||||
} else {
|
||||
System.out.println("Invalid arguments. You should provide a pair of jpack and directory to act as in/output.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,3 +8,5 @@ pluginManagement {
|
||||
gradlePluginPortal()
|
||||
}
|
||||
}
|
||||
|
||||
include 'bundler'
|
||||
@@ -1,6 +1,6 @@
|
||||
package io.ib67.sfcraft;
|
||||
|
||||
import io.ib67.sfcraft.bundler.Bundle;
|
||||
import io.ib67.sfcraft.bundler.BundleWriter;
|
||||
import io.ib67.sfcraft.config.JustBackupConfig;
|
||||
import io.ib67.sfcraft.mixin.LevelStorageAccessor;
|
||||
import io.ib67.sfcraft.strategy.BackupStrategy;
|
||||
@@ -12,9 +12,10 @@ import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.List;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import static io.ib67.sfcraft.JustBackupMod.PERFORMING_BACKUP;
|
||||
import static io.ib67.sfcraft.JustBackupMod.BACKUP_LOCK;
|
||||
|
||||
@Log4j2
|
||||
public record BackupWorker(
|
||||
@@ -30,11 +31,16 @@ public record BackupWorker(
|
||||
throw new IllegalStateException("MinecraftServer is not initialized yet, not doing backup");
|
||||
}
|
||||
int i = 0;
|
||||
while (!PERFORMING_BACKUP.compareAndSet(false, true)) {
|
||||
if (i++ % 10 == 0) log.info("Server is saving world while performing backup! waiting...");
|
||||
while (!BACKUP_LOCK.compareAndSet(IOState.IDLE, IOState.BACKUP)) {
|
||||
log.info("Server is saving world while performing backup! waiting...");
|
||||
i++;
|
||||
Thread.sleep(2000);
|
||||
}
|
||||
// performing backup is now true
|
||||
var backupName = "Backup_" + DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(LocalDateTime.now());
|
||||
var backupParentRoot = Path.of(config.temporaryBackupDir());
|
||||
if (Files.notExists(backupParentRoot)) Files.createDirectories(backupParentRoot);
|
||||
var backupFile = backupParentRoot.resolve(backupName + ".jpack");
|
||||
try {
|
||||
if (i == 0) {
|
||||
// not saved yet.
|
||||
@@ -42,22 +48,27 @@ public record BackupWorker(
|
||||
}
|
||||
var session = ((LevelStorageAccessor) server).getSession();
|
||||
var path = session.getDirectory().getRootPath();
|
||||
var backupName = "Backup_" + DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(LocalDateTime.now());
|
||||
var backupParentRoot = Path.of(config.temporaryBackupDir());
|
||||
if (Files.notExists(backupParentRoot)) Files.createDirectories(backupParentRoot);
|
||||
var backupFile = backupParentRoot.resolve(backupName + ".jpack");
|
||||
log.info("Creating bundle {}", backupName);
|
||||
//todo diff
|
||||
Path worldDir = Path.of(path);
|
||||
new Bundle(new WorldDir(worldDir))
|
||||
.compress(config.compress())
|
||||
.allowGunzip(config.allowGunzip())
|
||||
.buildBundle(backupFile);
|
||||
var tree = new WorldDir(worldDir);
|
||||
BundleWriter.createBundle(backupFile, tree.everything(), c -> c.allowGunzip(config.allowGunzip())
|
||||
.relativeRoot(worldDir)
|
||||
.compressionLevel(config.compressionLevel()));
|
||||
var backup = strategy.createBackup(worldDir, backupFile);
|
||||
Files.deleteIfExists(backupFile);
|
||||
return backup;
|
||||
} catch (Exception e) {
|
||||
Files.deleteIfExists(backupFile);
|
||||
throw e;
|
||||
} finally {
|
||||
PERFORMING_BACKUP.set(false);
|
||||
// Though we won't access the state within backup progress, there are certain cases
|
||||
// that this state will be changed to STORAGE_SYNC.
|
||||
// Also take a look at MixinRegionBasedStorage.
|
||||
while (!BACKUP_LOCK.compareAndSet(IOState.BACKUP, IOState.IDLE)) {
|
||||
log.error("Expect BACKUP state but got {}", BACKUP_LOCK.get());
|
||||
Thread.sleep(2000);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package io.ib67.sfcraft;
|
||||
|
||||
public enum IOState {
|
||||
SAVING_WORLD,
|
||||
BACKUP,
|
||||
STORAGE_SYNC,
|
||||
IDLE
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
package io.ib67.sfcraft;
|
||||
|
||||
import com.github.luben.zstd.Zstd;
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import com.google.gson.reflect.TypeToken;
|
||||
@@ -45,6 +46,8 @@ import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
import static net.minecraft.server.command.CommandManager.argument;
|
||||
import static net.minecraft.server.command.CommandManager.literal;
|
||||
@@ -57,7 +60,7 @@ public class JustBackupMod implements ModInitializer {
|
||||
* 1. A minecraft auto-save is in-progress.
|
||||
* 2. The backup is in-progress
|
||||
*/
|
||||
public static final AtomicBoolean PERFORMING_BACKUP = new AtomicBoolean(false);
|
||||
public static final AtomicReference<IOState> BACKUP_LOCK = new AtomicReference<>(IOState.IDLE);
|
||||
public static final Gson GSON = new GsonBuilder().setPrettyPrinting()
|
||||
.registerTypeAdapter(StorageOption.class, new StorageOptionSerializer(
|
||||
Map.of("s3", StorageOption.S3.class,
|
||||
@@ -75,8 +78,9 @@ public class JustBackupMod implements ModInitializer {
|
||||
@SneakyThrows
|
||||
public void onInitialize() {
|
||||
LOGGER.info("Loading backup configuration");
|
||||
backupIndexPath = Path.of("config").resolve("just_backup_index.json");
|
||||
var configPath = Path.of("config").resolve("just_backup.json");
|
||||
Path config1 = Path.of("config");
|
||||
backupIndexPath = config1.resolve("just_backup_index.json");
|
||||
var configPath = config1.resolve("just_backup.json");
|
||||
if (Files.notExists(configPath.getParent()))
|
||||
Files.createDirectories(configPath.getParent());
|
||||
if (Files.notExists(configPath))
|
||||
@@ -131,7 +135,7 @@ public class JustBackupMod implements ModInitializer {
|
||||
private void onBackupComplete(@UnknownNullability Backup backup, @UnknownNullability Throwable throwable) {
|
||||
if (throwable != null) {
|
||||
server.getPlayerManager().broadcast(Text.literal("Backup failed. For administrators, please check your server console."), false);
|
||||
throwable.printStackTrace();
|
||||
LOGGER.error(throwable.getMessage(), throwable);
|
||||
} else {
|
||||
Files.writeString(backupIndexPath, GSON.toJson(tracker.getTrackedBackups()));
|
||||
server.getPlayerManager().broadcast(Text.literal("Backup success. " + backup), false);
|
||||
@@ -151,7 +155,7 @@ public class JustBackupMod implements ModInitializer {
|
||||
var cfg = new JustBackupConfig(
|
||||
60,
|
||||
5,
|
||||
true,
|
||||
Zstd.defaultCompressionLevel(),
|
||||
true,
|
||||
"./.backup_cache",
|
||||
new StorageOption.Local("backups", 0)
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
|
||||
import io.ib67.sfcraft.WorldDir;
|
||||
import lombok.Data;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.experimental.Accessors;
|
||||
import org.apache.commons.lang3.Validate;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.List;
|
||||
|
||||
@Accessors(fluent = true)
|
||||
@Data
|
||||
public class Bundle {
|
||||
public static final int MAGIC = 0xBACEBACE;
|
||||
public static final int COMPRESSED_MAGIC = 0xBACEAAAA;
|
||||
protected final WorldDir dir;
|
||||
protected boolean compress = true;
|
||||
protected boolean allowGunzip = true;
|
||||
protected boolean allowParallel = false;
|
||||
|
||||
|
||||
public Bundle(WorldDir dir) {
|
||||
this.dir = dir;
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
public static void unbundleFile(Path bundle, Path out) {
|
||||
try (var bundlerReader = new SerialBundlerReader(bundle)) {
|
||||
while (bundlerReader.hasEntry()) {
|
||||
var entry = bundlerReader.readEntry();
|
||||
var target = out.resolve(entry.fileName());
|
||||
Files.createDirectories(target.getParent());
|
||||
bundlerReader.writeTo(FileChannel.open(target, StandardOpenOption.WRITE,
|
||||
StandardOpenOption.CREATE_NEW), entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
public void buildBundle(Path backupFile) {
|
||||
checkCondition();
|
||||
var sortedEntries = dir.everything().stream().sorted((a, b) -> {
|
||||
boolean aMatch = a.endsWith(".mca") || a.endsWith(".dat");
|
||||
boolean bMatch = b.endsWith(".mca") || b.endsWith(".dat");
|
||||
|
||||
if (aMatch && !bMatch) return -1;
|
||||
if (!aMatch && bMatch) return 1;
|
||||
return a.compareTo(b);
|
||||
}).map(dir.root()::relativize).toList();
|
||||
if (Files.notExists(backupFile.getParent()))
|
||||
Files.createDirectories(backupFile.getParent());
|
||||
SerialBundlerWriter writer;
|
||||
if (compress) {
|
||||
var fout = Files.newOutputStream(backupFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
|
||||
writeMagic(fout, null, true);
|
||||
var out = new ZstdOutputStreamNoFinalizer(fout);
|
||||
writer = new SerialBundlerWriter(out);
|
||||
} else {
|
||||
var chan = FileChannel.open(backupFile);
|
||||
writer = new SerialBundlerWriter(chan, 0);
|
||||
writeMagic(null, chan, false);
|
||||
}
|
||||
// todo parallelism
|
||||
writeEntries(sortedEntries, writer, allowGunzip && compress);
|
||||
writer.close();
|
||||
|
||||
try (var backup = FileChannel.open(backupFile, StandardOpenOption.WRITE, StandardOpenOption.APPEND);
|
||||
var bos = new ByteArrayOutputStream();
|
||||
var metadata = new EntryOutputStream(new ZstdOutputStreamNoFinalizer(bos))) {
|
||||
var entries = writer.getEntries();
|
||||
for (BundlerEntry entry : entries) {
|
||||
metadata.write(entry);
|
||||
}
|
||||
metadata.flush();
|
||||
var result = bos.toByteArray();
|
||||
var buffer = ByteBuffer.allocate(result.length +4 + 4);
|
||||
buffer.put(result);
|
||||
buffer.putInt(entries.size());
|
||||
buffer.putInt(result.length);
|
||||
writeFully(backup, buffer);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static void writeMagic(OutputStream out, FileChannel channel, boolean compress) throws IOException {
|
||||
var magic = compress ? Bundle.COMPRESSED_MAGIC : Bundle.MAGIC;
|
||||
if (channel != null) {
|
||||
channel.write(ByteBuffer.allocate(4).putInt(magic).flip());
|
||||
} else {
|
||||
out.write(ByteBuffer.allocate(4).putInt(magic).array());
|
||||
}
|
||||
}
|
||||
|
||||
private void writeFully(FileChannel chan, ByteBuffer buf) throws IOException {
|
||||
buf.flip();
|
||||
while (buf.hasRemaining()) {
|
||||
chan.write(buf);
|
||||
}
|
||||
}
|
||||
|
||||
private void checkCondition() {
|
||||
if (allowParallel) {
|
||||
Validate.isTrue(!compress && !allowGunzip);
|
||||
}
|
||||
}
|
||||
|
||||
private void writeEntries(
|
||||
List<Path> sortedEntries,
|
||||
SerialBundlerWriter writer,
|
||||
boolean allowGunzip) throws IOException {
|
||||
for (Path sortedEntry : sortedEntries) {
|
||||
var fileName = sortedEntry.getFileName();
|
||||
var shouldGunzip = allowGunzip && (fileName.endsWith("mca") || fileName.endsWith("dat"));
|
||||
if (shouldGunzip) {
|
||||
writer.writeGunzip(dir.root().resolve(sortedEntry), sortedEntry.toString());
|
||||
} else {
|
||||
writer.writePlain(dir.root().resolve(sortedEntry), sortedEntry.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
public record BundlerEntry(
|
||||
String fileName,
|
||||
boolean unGzipped,
|
||||
long fileLength,
|
||||
long offset
|
||||
) {
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
import java.io.FilterInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
public class EntryInputStream extends FilterInputStream {
|
||||
/**
|
||||
* Creates a {@code FilterInputStream}
|
||||
* by assigning the argument {@code in}
|
||||
* to the field {@code this.in} so as
|
||||
* to remember it for later use.
|
||||
*
|
||||
* @param in the underlying input stream, or {@code null} if
|
||||
* this instance is to be created without an underlying stream.
|
||||
*/
|
||||
protected EntryInputStream(InputStream in) {
|
||||
super(in);
|
||||
}
|
||||
|
||||
public BundlerEntry readEntry() throws IOException {
|
||||
var len = ByteBuffer.wrap(readNBytes(4)).getInt();
|
||||
var gunzipped = len < 0;
|
||||
len = Math.abs(len);
|
||||
var name = new String(readNBytes(len)).intern();
|
||||
var buf = ByteBuffer.wrap(readNBytes(8 + 8));
|
||||
var fileLen = buf.getLong();
|
||||
var fileOffset = buf.getLong();
|
||||
return new BundlerEntry(name, gunzipped, fileLen, fileOffset);
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
import java.io.FilterOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
public class EntryOutputStream extends FilterOutputStream {
|
||||
/**
|
||||
* Creates an output stream filter built on top of the specified
|
||||
* underlying output stream.
|
||||
*
|
||||
* @param out the underlying output stream to be assigned to
|
||||
* the field {@code this.out} for later use, or
|
||||
* {@code null} if this instance is to be
|
||||
* created without an underlying stream.
|
||||
*/
|
||||
public EntryOutputStream(OutputStream out) {
|
||||
super(out);
|
||||
}
|
||||
|
||||
public void write(BundlerEntry entry) throws IOException {
|
||||
var fileName = entry.fileName().getBytes(StandardCharsets.UTF_8);
|
||||
var buf = ByteBuffer.allocate(4).putInt(entry.unGzipped() ? fileName.length * -1 : fileName.length);
|
||||
out.write(buf.array());
|
||||
out.write(fileName);
|
||||
buf = ByteBuffer.allocate(8 + 8);
|
||||
buf.putLong(entry.fileLength());
|
||||
buf.putLong(entry.offset());
|
||||
out.write(buf.array());
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
import io.ib67.kiwi.ArgOpts;
|
||||
import io.ib67.sfcraft.WorldDir;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) {
|
||||
var opts = ArgOpts.builder()
|
||||
.args(args).description("Tools for .jpack format bundle").programName("jpack").build();
|
||||
var in = opts.string("in", "Input. Can be directory or jpack bundle", null);
|
||||
var out = opts.string("out", "Output. Can be jpack bundle or directory", null);
|
||||
var help = opts.bool("help", false);
|
||||
var allowGunzip = opts.bool("gunzip",
|
||||
"Should we recompress NBT files for maximum compression rate (depends on compress)", true);
|
||||
var compress = opts.bool("compress", "Should we compress?", true);
|
||||
var parallel = opts.bool("parallel", "Use parallelized bundler. This conflicts with compress options", false);
|
||||
if (help) {
|
||||
opts.printHelp(System.out);
|
||||
return;
|
||||
}
|
||||
if (allowGunzip && !compress) {
|
||||
System.out.println("allowGunzip depends on compress feature.");
|
||||
return;
|
||||
}
|
||||
if (in == null || out == null) {
|
||||
System.out.println("Both in and out should be provided");
|
||||
return;
|
||||
}
|
||||
var inPath = Path.of(in);
|
||||
var outPath = Path.of(out);
|
||||
if (!Files.exists(inPath)) {
|
||||
System.out.println("The input isn't exists.");
|
||||
return;
|
||||
}
|
||||
if (Files.exists(outPath)) {
|
||||
System.out.println("The output is already exists. Please move it to other places");
|
||||
return;
|
||||
}
|
||||
if (in.endsWith(".jpack")) {
|
||||
Bundle.unbundleFile(inPath, outPath);
|
||||
} else if (out.endsWith(".jpack")) {
|
||||
if (!Files.isDirectory(inPath))
|
||||
throw new IllegalArgumentException("Your input should be a world directory.");
|
||||
new Bundle(new WorldDir(inPath))
|
||||
.allowGunzip(allowGunzip)
|
||||
.compress(compress)
|
||||
.allowParallel(parallel)
|
||||
.buildBundle(outPath);
|
||||
} else {
|
||||
System.out.println("Invalid arguments. You should provide a pair of jpack and directory to act as in/output.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
import com.github.luben.zstd.Zstd;
|
||||
import com.github.luben.zstd.ZstdInputStreamNoFinalizer;
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.Channels;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Objects;
|
||||
|
||||
public class SerialBundlerReader implements Closeable {
|
||||
protected final InputStream in;
|
||||
protected final FileChannel inChannel; // for direct copy
|
||||
|
||||
protected final EntryInputStream metadataIn;
|
||||
protected final byte[] buffer = new byte[4096];
|
||||
protected final RandomAccessFile raf;
|
||||
protected boolean entryRead;
|
||||
protected int entries;
|
||||
|
||||
// public SerialBundlerReader(InputStream in, InputStream metadataIn) {
|
||||
// this.in = Objects.requireNonNull(in);
|
||||
// this.metadataIn = new EntryInputStream(Objects.requireNonNull(metadataIn));
|
||||
// inChannel = null;
|
||||
// }
|
||||
|
||||
public SerialBundlerReader(Path path) throws IOException {
|
||||
raf = new RandomAccessFile(path.toFile(), "r");
|
||||
var totalLen = Files.size(path);
|
||||
var magic = raf.readInt();
|
||||
var compressed = magic == Bundle.COMPRESSED_MAGIC;
|
||||
if (!compressed && magic != Bundle.MAGIC) throw new IOException("Invalid bundle file");
|
||||
|
||||
// read length
|
||||
raf.seek(totalLen - 4);
|
||||
var metadataLength = raf.readInt();
|
||||
raf.seek(totalLen - 8);
|
||||
entries = raf.readInt();
|
||||
// read metadata
|
||||
var metadataBegin = totalLen - 8 - metadataLength;
|
||||
var metadataBytes = new byte[metadataLength];
|
||||
raf.seek(metadataBegin);
|
||||
raf.readFully(metadataBytes);
|
||||
|
||||
this.metadataIn = new EntryInputStream(new ZstdInputStreamNoFinalizer(new ByteArrayInputStream(metadataBytes)));
|
||||
// check compressed
|
||||
raf.seek(4);
|
||||
var fis = Channels.newInputStream(raf.getChannel());
|
||||
if (compressed) {
|
||||
in = new ZstdInputStreamNoFinalizer(fis);
|
||||
inChannel = null;
|
||||
} else {
|
||||
in = fis;
|
||||
inChannel = raf.getChannel();
|
||||
}
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
public boolean hasEntry() {
|
||||
return entries > 0;
|
||||
}
|
||||
|
||||
public BundlerEntry readEntry() throws IOException {
|
||||
if (!hasEntry()) throw new IOException("All entries are read");
|
||||
var entry = metadataIn.readEntry();
|
||||
entryRead = true;
|
||||
entries--;
|
||||
return entry;
|
||||
}
|
||||
|
||||
public void writeTo(FileChannel channel, BundlerEntry entry) throws IOException {
|
||||
if (inChannel != null) {
|
||||
inChannel.transferTo(entry.offset(), entry.fileLength(), channel);
|
||||
} else {
|
||||
writeTo(Channels.newOutputStream(channel), entry);
|
||||
}
|
||||
}
|
||||
|
||||
public void writeTo(OutputStream out, BundlerEntry entry) throws IOException {
|
||||
Objects.requireNonNull(entry);
|
||||
var remaining = entry.fileLength();
|
||||
var buffer = this.buffer;
|
||||
var totalRead = 0;
|
||||
do {
|
||||
var read = in.read(buffer, 0, (int) Math.min(remaining - totalRead, buffer.length));
|
||||
out.write(buffer, 0, read);
|
||||
totalRead += read;
|
||||
} while (totalRead < remaining);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
in.close();
|
||||
metadataIn.close();
|
||||
raf.close();
|
||||
}
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
package io.ib67.sfcraft.bundler;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.channels.Channels;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
@AllArgsConstructor
|
||||
public class SerialBundlerWriter implements Closeable {
|
||||
protected final FileChannel channel;
|
||||
protected final OutputStream out;
|
||||
protected long offset;
|
||||
@Getter
|
||||
protected List<BundlerEntry> entries;
|
||||
|
||||
public SerialBundlerWriter(OutputStream out) {
|
||||
Objects.requireNonNull(out);
|
||||
this.out = out;
|
||||
this.entries = new ArrayList<>();
|
||||
channel = null;
|
||||
}
|
||||
|
||||
public SerialBundlerWriter(FileChannel channel, long offset) {
|
||||
Objects.requireNonNull(channel);
|
||||
this.offset = offset;
|
||||
this.entries = new ArrayList<>();
|
||||
this.channel = channel;
|
||||
out = null;
|
||||
}
|
||||
|
||||
public void writeGunzip(Path path, String fileName) throws IOException {
|
||||
try (var gzip = new GZIPInputStream(Files.newInputStream(path, StandardOpenOption.READ))) {
|
||||
var len = channel != null ? gzip.transferTo(Channels.newOutputStream(channel)) :
|
||||
gzip.transferTo(out);
|
||||
entries.add(new BundlerEntry(fileName, true, len, offset));
|
||||
offset += len;
|
||||
}
|
||||
}
|
||||
|
||||
public void writePlain(Path path, String fileName) throws IOException {
|
||||
if (channel != null) {
|
||||
writeZC(path, fileName);
|
||||
} else {
|
||||
var current = offset;
|
||||
try (var in = Files.newInputStream(path)) {
|
||||
in.transferTo(out);
|
||||
}
|
||||
entries.add(new BundlerEntry(fileName, false, Files.size(path), current));
|
||||
}
|
||||
}
|
||||
|
||||
private void writeZC(Path path, String fileName) throws IOException {
|
||||
try (var in = FileChannel.open(path, StandardOpenOption.READ)) {
|
||||
var begin = offset;
|
||||
var currentPosition = offset;
|
||||
long transferred = 0;
|
||||
while ((transferred = in.transferTo(currentPosition, in.size() - transferred, channel)) != 0) {
|
||||
currentPosition += transferred;
|
||||
}
|
||||
offset = currentPosition;
|
||||
entries.add(new BundlerEntry(
|
||||
fileName, false, offset - begin, begin
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
if (out != null) out.close();
|
||||
if (channel != null) channel.close();
|
||||
}
|
||||
}
|
||||
@@ -7,14 +7,15 @@ import java.util.List;
|
||||
public record JustBackupConfig(
|
||||
long backupIntervalMinutes,
|
||||
int keepBackups,
|
||||
int compressionLevel,
|
||||
boolean allowGunzip,
|
||||
boolean compress,
|
||||
String temporaryBackupDir,
|
||||
StorageOption option
|
||||
) {
|
||||
public JustBackupConfig {
|
||||
Validate.isTrue(backupIntervalMinutes > 0, "backup interval must be positive");
|
||||
Validate.isTrue(keepBackups > 0, "keep backups must be positive");
|
||||
Validate.isTrue(compressionLevel > 0, "compression level must be positive");
|
||||
Validate.notNull(option, "option cannot be null");
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
package io.ib67.sfcraft.mixin;
|
||||
|
||||
import io.ib67.sfcraft.IOState;
|
||||
import io.ib67.sfcraft.JustBackupMod;
|
||||
import lombok.Getter;
|
||||
import net.minecraft.server.MinecraftServer;
|
||||
@@ -16,18 +17,30 @@ import java.util.concurrent.atomic.AtomicBoolean;
|
||||
@Mixin(MinecraftServer.class)
|
||||
public abstract class MixinMinecraftServer {
|
||||
|
||||
@Shadow private int ticksUntilAutosave;
|
||||
@Shadow
|
||||
private int ticksUntilAutosave;
|
||||
|
||||
@Shadow protected abstract int getAutosaveInterval();
|
||||
@Shadow
|
||||
protected abstract int getAutosaveInterval();
|
||||
|
||||
@Inject(method = "runAutosave", at = @At("HEAD"), cancellable = true)
|
||||
private void backup$checkSaving(CallbackInfo ci) {
|
||||
// is already backing up
|
||||
this.ticksUntilAutosave = getAutosaveInterval();
|
||||
if (!JustBackupMod.PERFORMING_BACKUP.compareAndSet(false, true)) ci.cancel();
|
||||
IOState witness;
|
||||
while ((witness = JustBackupMod.BACKUP_LOCK.compareAndExchange(IOState.IDLE, IOState.SAVING_WORLD)) == IOState.STORAGE_SYNC) {
|
||||
try {
|
||||
Thread.sleep(1000);
|
||||
} catch (InterruptedException e) {
|
||||
System.err.println("Error while waiting for chunk sync. " + e);
|
||||
ci.cancel();
|
||||
}
|
||||
}
|
||||
if (witness == IOState.BACKUP) ci.cancel();
|
||||
}
|
||||
|
||||
@Inject(method = "runAutosave", at = @At("RETURN"))
|
||||
private void backup$disableProtect(CallbackInfo ci) {
|
||||
JustBackupMod.PERFORMING_BACKUP.set(false);
|
||||
JustBackupMod.BACKUP_LOCK.setRelease(IOState.IDLE);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package io.ib67.sfcraft.mixin;
|
||||
|
||||
import io.ib67.sfcraft.IOState;
|
||||
import io.ib67.sfcraft.JustBackupMod;
|
||||
import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectIterator;
|
||||
import lombok.SneakyThrows;
|
||||
import net.minecraft.world.storage.RegionBasedStorage;
|
||||
import net.minecraft.world.storage.RegionFile;
|
||||
import org.spongepowered.asm.mixin.Final;
|
||||
import org.spongepowered.asm.mixin.Mixin;
|
||||
import org.spongepowered.asm.mixin.Overwrite;
|
||||
import org.spongepowered.asm.mixin.Shadow;
|
||||
import org.spongepowered.asm.mixin.injection.At;
|
||||
import org.spongepowered.asm.mixin.injection.Inject;
|
||||
import org.spongepowered.asm.mixin.injection.callback.CallbackInfo;
|
||||
|
||||
@Mixin(RegionBasedStorage.class)
|
||||
public abstract class MixinRegionBasedStorage {
|
||||
@Shadow
|
||||
@Final
|
||||
private Long2ObjectLinkedOpenHashMap<RegionFile> cachedRegionFiles;
|
||||
|
||||
/**
|
||||
* @author iceBear67
|
||||
* @reason to wrap the saving process with an exception handler.
|
||||
*/
|
||||
@Overwrite
|
||||
@SneakyThrows
|
||||
public void sync() {
|
||||
// allowed states:
|
||||
// STORAGE_SYNC, AUTO_SAVE, IDLE
|
||||
// we try
|
||||
var witness = JustBackupMod.BACKUP_LOCK.getAndSet(IOState.STORAGE_SYNC);
|
||||
if (witness == IOState.BACKUP) {
|
||||
// yield out and keep blocking.
|
||||
// The backup worker will wait for us setting this back to backup.
|
||||
JustBackupMod.BACKUP_LOCK.set(IOState.BACKUP);
|
||||
while (JustBackupMod.BACKUP_LOCK.getAcquire() == IOState.BACKUP) {
|
||||
Thread.sleep(2000);
|
||||
}
|
||||
}
|
||||
// program point: IDLE, AUTOSAVE, STORAGESYNC
|
||||
try {
|
||||
for (RegionFile value : this.cachedRegionFiles.values()) {
|
||||
value.sync();
|
||||
}
|
||||
} finally {
|
||||
// we asserts that there are no other threads calling sync()
|
||||
// so we can just jump back to the witness value.
|
||||
// witness: AUTO_SAVE or IDLE
|
||||
JustBackupMod.BACKUP_LOCK.compareAndExchange(IOState.STORAGE_SYNC, witness);
|
||||
JustBackupMod.BACKUP_LOCK.setRelease(witness);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,6 @@
|
||||
package io.ib67.sfcraft.strategy;
|
||||
|
||||
import io.ib67.sfcraft.Backup;
|
||||
import io.ib67.sfcraft.WorldDir;
|
||||
import io.ib67.sfcraft.bundler.Bundle;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
|
||||
@@ -1,26 +1,15 @@
|
||||
package io.ib67.sfcraft.strategy;
|
||||
|
||||
import com.github.luben.zstd.Zstd;
|
||||
import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
|
||||
import io.ib67.sfcraft.Backup;
|
||||
import io.ib67.sfcraft.WorldDir;
|
||||
import io.ib67.sfcraft.bundler.*;
|
||||
import io.ib67.sfcraft.bundler.BundleReader;
|
||||
import io.ib67.sfcraft.config.StorageOption;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.apache.commons.lang3.Validate;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.*;
|
||||
|
||||
@Log4j2
|
||||
public class LocalBackupStrategy implements BackupStrategy {
|
||||
@@ -51,7 +40,7 @@ public class LocalBackupStrategy implements BackupStrategy {
|
||||
// check if it is compressed
|
||||
var backupFile = backupParentRoot.resolve(backup.backupKey());
|
||||
if (Files.notExists(backupFile)) throw new IllegalArgumentException("Backup " + backupFile + " does not exist");
|
||||
Bundle.unbundleFile(backupFile, restorePath);
|
||||
BundleReader.extract(backupFile, restorePath);
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package io.ib67.sfcraft.strategy;
|
||||
|
||||
import io.ib67.sfcraft.Backup;
|
||||
import io.ib67.sfcraft.bundler.Bundle;
|
||||
import io.ib67.sfcraft.bundler.BundleReader;
|
||||
import io.ib67.sfcraft.config.StorageOption;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
@@ -84,7 +84,7 @@ public class S3BackupStrategy implements BackupStrategy {
|
||||
var target = temporaryDownloadPath.resolve("s3_" + System.currentTimeMillis());
|
||||
try (var fs = Files.newOutputStream(target, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
|
||||
resp.transferTo(fs);
|
||||
Bundle.unbundleFile(target, restorePath);
|
||||
BundleReader.extract(target, restorePath);
|
||||
} finally {
|
||||
Files.deleteIfExists(target);
|
||||
}
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
"compatibilityLevel": "JAVA_21",
|
||||
"mixins": [
|
||||
"LevelStorageAccessor",
|
||||
"MixinMinecraftServer"
|
||||
"MixinMinecraftServer",
|
||||
"MixinRegionBasedStorage"
|
||||
],
|
||||
"injectors": {
|
||||
"defaultRequire": 1
|
||||
|
||||
Reference in new issue
Block a user