fix fix fix

This commit is contained in:
iceBear67 committed 2025-10-01 16:59:17 +08:00
1 parent 394a4d986d
commit 48bf045825
16 files changed
+172 -59

No files matched your search

+3 -1
View File
@@ -67,13 +67,15 @@ dependencies {
shadow project(":bundler") shadow project(":bundler")
shadow 'software.amazon.awssdk:s3:2.34.0' shadow 'software.amazon.awssdk:s3:2.34.0'
} }
configurations.implementation.extendsFrom(configurations.shadow) configurations.implementation.extendsFrom(configurations.shadow)
shadowJar { shadowJar {
configurations = [project.configurations.shadow] configurations = [project.configurations.shadow]
exclude("META-INF") exclude("META-INF")
dependencies { dependencies {
exclude(dependency("software.amazon.awssdk:aws-xml-protocol:.*")) exclude(dependency("software.amazon.awssdk:aws-xml-protocol:.*"))
exclude(dependency("org.lz4:lz4-java:.*"))
exclude(dependency("it.unimi.dsi:fastutil:.*"))
exclude(dependency("io.netty:netty-all:.*"))
} }
minimize() minimize()
//relocate 'com.google', 'sfcraft.relocation.google' //relocate 'com.google', 'sfcraft.relocation.google'
+8 -7
View File
@@ -8,18 +8,19 @@ plugins {
dependencies { dependencies {
api 'com.github.luben:zstd-jni:1.5.7-4' api 'com.github.luben:zstd-jni:1.5.7-4'
compileOnly 'io.netty:netty-all:4.2.6.Final' testCompileOnly 'io.netty:netty-all:4.2.6.Final'
api "io.ib67.kiwi:lang:1.2.3-SNAPSHOT" api "io.ib67.kiwi:lang:1.2.3-SNAPSHOT"
compileOnly 'io.netty:netty-all:4.2.6.Final' implementation 'io.netty:netty-all:4.2.6.Final'
compileOnly 'it.unimi.dsi:fastutil:8.5.15' implementation 'it.unimi.dsi:fastutil:8.5.15'
compileOnly 'org.lz4:lz4-java:1.8.0' implementation 'org.lz4:lz4-java:1.8.0'
compileOnly 'org.lz4:lz4-java:1.8.0'
testImplementation 'org.junit.jupiter:junit-jupiter:5.13.4' testImplementation 'org.junit.jupiter:junit-jupiter:5.13.4'
testRuntimeOnly 'org.junit.platform:junit-platform-launcher' testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
} }
shadowJar { configurations {
minimize() testRuntimeOnly {
extendsFrom compileOnly
}
} }
test { test {
@@ -62,12 +62,18 @@ public class BundleReader {
entryData.writeBytes(buffer, 0, read); entryData.writeBytes(buffer, 0, read);
} }
try (var reassembler = new RegionParser(entryData, allocator)) { try (var reassembler = new RegionParser(entryData, allocator)) {
var result = reassembler.writeReassembled(RegionFile.CompressType.GZIP); var result = reassembler.writeReassembled(RegionFile.CompressType.ZLIB);
try (var fc = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { try (var fc = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
result.readBytes(fc, 0, result.readableBytes()); result.readBytes(fc, 0, result.readableBytes());
} finally { } finally {
result.release(); result.release();
} }
} catch (Exception e) {
try (var ch = FileChannel.open(Path.of("dump.mca"), StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
entryData.readerIndex(0);
entryData.readBytes(ch, 0, entryData.readableBytes());
}
throw new IOException("Error while reassembling " + entry.name(), e);
} }
} else { } else {
writePlain(path, entry, in); writePlain(path, entry, in);
@@ -75,7 +75,7 @@ public class BundleWriter implements Closeable {
var futures = new ArrayList<CompletableFuture<Path>>(); var futures = new ArrayList<CompletableFuture<Path>>();
for (int i = 0; i < parted.size(); i++) { for (int i = 0; i < parted.size(); i++) {
var pathList = parted.get(i); var pathList = parted.get(i);
var output = parentOfBundle.resolve("Worker_" + System.currentTimeMillis() + "_" + i + ".jbp.zst"); var output = parentOfBundle.resolve("Worker_" + System.currentTimeMillis() + "_" + i + ".swb.zst");
var future = CompletableFuture.supplyAsync(() -> createBundleWorker(pathList, output, config), executor); var future = CompletableFuture.supplyAsync(() -> createBundleWorker(pathList, output, config), executor);
futures.add(future); futures.add(future);
} }
@@ -142,7 +142,7 @@ public class BundleWriter implements Closeable {
} }
public void write(Path path, boolean suggestGunzip) throws IOException { public void write(Path path, boolean suggestGunzip) throws IOException {
if (suggestGunzip) { if (suggestGunzip && allowReassemble) {
//todo size limit //todo size limit
writeReassembleInMem(path); writeReassembleInMem(path);
return; return;
@@ -158,22 +158,26 @@ public class BundleWriter implements Closeable {
return; return;
} }
if(verbose) System.out.println("Reassembling " + path); if(verbose) System.out.println("Reassembling " + path);
var uncompressed = allocator.buffer(); var finalResult = allocator.buffer();
var rPath = relativeRoot.toAbsolutePath().relativize(path.toAbsolutePath()); var rPath = relativeRoot.toAbsolutePath().relativize(path.toAbsolutePath());
try (var out = new EntryOutputStream(new ByteBufOutputStream(uncompressed)); try (var out = new EntryOutputStream(new ByteBufOutputStream(finalResult));
var rf = new RegionParser(path, allocator)) { var rf = new RegionParser(path, allocator)) {
// uncompressed result
var result = rf.writeReassembled(RegionFile.CompressType.NONE); var result = rf.writeReassembled(RegionFile.CompressType.NONE);
var length = result.readableBytes(); var length = result.readableBytes();
out.writeEntryHeader(rPath.toString(), BundleEntry.ATTR_REASSEMBLE, length, System.currentTimeMillis()); out.writeEntryHeader(rPath.toString(), BundleEntry.ATTR_REASSEMBLE, length, System.currentTimeMillis());
result.readBytes(out, length); result.readBytes(out, length);
result.release(); result.release();
uncompressed.readBytes(outputStream, uncompressed.readableBytes()); // all done
} catch (IOException e) { } catch (IOException e) {
System.err.println("Failed to parse region " + path + ": " + e); System.err.println("Failed to parse region " + path + ": " + e);
finalResult.release();
writePlain(path); writePlain(path);
} finally { return;
uncompressed.release();
} }
finalResult.readBytes(outputStream, finalResult.readableBytes());
finalResult.release();
} }
private void writePlain(Path path) throws IOException { private void writePlain(Path path) throws IOException {
@@ -186,6 +190,9 @@ public class BundleWriter implements Closeable {
while ((read = fs.read(buffer)) > 0) { while ((read = fs.read(buffer)) > 0) {
outputStream.write(buffer, 0, read); outputStream.write(buffer, 0, read);
} }
}catch(IOException e){
System.err.println("Error while bundling "+path);
throw e;
} }
} }
@@ -26,7 +26,7 @@ public class Main {
@SneakyThrows @SneakyThrows
public static void main(String[] args) { public static void main(String[] args) {
var opts = ArgOpts.builder() var opts = ArgOpts.builder()
.args(args).description("Tools for .jbp.zst format bundle").programName("jpack").build(); .args(args).description("Tools for .swb.zst format bundle").programName("jpack").build();
var in = opts.string("in", "Input. Can be directory or jpack bundle", null); var in = opts.string("in", "Input. Can be directory or jpack bundle", null);
var out = opts.string("out", "Output. Can be jpack bundle or directory", null); var out = opts.string("out", "Output. Can be jpack bundle or directory", null);
var help = opts.bool("help", false); var help = opts.bool("help", false);
@@ -81,9 +81,9 @@ public class Main {
return; return;
} }
if (in.endsWith(".jbp.zst")) { if (in.endsWith(".swb.zst")) {
BundleReader.builder().build().extract(inPath, outPath); BundleReader.builder().build().extract(inPath, outPath);
} else if (out.endsWith(".jbp.zst")) { } else if (out.endsWith(".swb.zst")) {
UnaryOperator<BundleWriter.BundleWriterBuilder> cfg = it -> it UnaryOperator<BundleWriter.BundleWriterBuilder> cfg = it -> it
.allowGunzip(allowGunzip) .allowGunzip(allowGunzip)
.relativeRoot(inPath) .relativeRoot(inPath)
@@ -89,11 +89,14 @@ public class RegionFile implements Closeable {
} }
public ByteBuf writeOutput() { public ByteBuf writeOutput() {
var sect = allocator.buffer(8192); var sect = allocator.buffer(4096);
sect.setZero(0, 4096);
sect.writerIndex(4096);
var compositeByteBuf = allocator.compositeBuffer(2 + regionData.size()); var compositeByteBuf = allocator.compositeBuffer(2 + regionData.size());
var beginSector = 2; var beginSector = 2;
compositeByteBuf.addComponent(true, sect); compositeByteBuf.addComponent(true, sect);
timestamp.retain(); timestamp.retain();
timestamp.readerIndex(0);
compositeByteBuf.addComponent(true, timestamp); compositeByteBuf.addComponent(true, timestamp);
for (var entry : regionData.short2ObjectEntrySet()) { for (var entry : regionData.short2ObjectEntrySet()) {
var chunkKey = entry.getShortKey(); var chunkKey = entry.getShortKey();
@@ -102,12 +105,21 @@ public class RegionFile implements Closeable {
var chunk = entry.getValue(); var chunk = entry.getValue();
var size = chunk.readableBytes(); var size = chunk.readableBytes();
var sizeSectors = Math.ceilDiv(size, 4096); var sizeSectors = Math.ceilDiv(size, 4096);
var zeros = 4096 - (size % 4096);
for (int i = 0; i < zeros; i++) {
chunk.writeByte(0);
}
if (sizeSectors > 255) { if (sizeSectors > 255) {
throw new IllegalArgumentException("MCC not supported."); throw new IllegalArgumentException("MCC not supported.");
} }
var indexEntry = Integer.reverseBytes(beginSector) << 1 | (sizeSectors & 0xFF); if(sizeSectors < 0) {
// sect = int[1024] throw new IllegalArgumentException("Invalid sizeSectors: "+sizeSectors);
sect.setInt((x + 32 * z) * 4, indexEntry); }
var begin = (x + 32 * z) * 4;
sect.setByte(begin, (beginSector >>> 16) & 0xFF);
sect.setByte(begin+1, (beginSector >>> 8) & 0xFF);
sect.setByte(begin+2, beginSector & 0xFF);
sect.setByte(begin+3, sizeSectors & 0xFF);
chunk.retain(); // retain a local reference chunk.retain(); // retain a local reference
compositeByteBuf.addComponent(true, chunk); compositeByteBuf.addComponent(true, chunk);
beginSector += sizeSectors; beginSector += sizeSectors;
@@ -22,12 +22,12 @@ public class RegionParser implements Closeable {
this.allocator = allocator; this.allocator = allocator;
var size = (int) Files.size(file); var size = (int) Files.size(file);
if (size < 8192) throw new IOException("The region file " + file + " has been truncated"); if (size < 8192) throw new IOException("The region file " + file + " has been truncated");
if(size > 8388608 * 2) throw new IOException("The region file " + file + " is too large"); //todo bug if (size > 8388608 * 2) throw new IOException("The region file " + file + " is too large"); //todo bug
source = allocator.buffer(size); source = allocator.buffer(size);
try (var fc = FileChannel.open(file, StandardOpenOption.READ)) { try (var fc = FileChannel.open(file, StandardOpenOption.READ)) {
var toRead = size; var toRead = size;
while(toRead > 0){ while (toRead > 0) {
toRead -= source.writeBytes(fc, 0L, (int) size); toRead -= source.writeBytes(fc, 0L, (int) size);
} }
} }
@@ -66,6 +66,7 @@ public class RegionParser implements Closeable {
if (compressType < 0) { if (compressType < 0) {
// mcc todo // mcc todo
} }
if (sectorLen == 0) return compressType;
buf.writeBytes(source, sectorLen - 1); buf.writeBytes(source, sectorLen - 1);
return compressType; return compressType;
} }
@@ -7,16 +7,32 @@ import io.netty.buffer.Unpooled;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.io.IOException; import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
public class TestRegionReassembler { public class TestRegionReassembler {
@Test @Test
public void testReassembler() throws IOException { public void testReassembler() throws IOException {
testMethod("region/r.9.-5.mca");
testMethod("region/r.-1.-1.mca");
}
private static void testMethod(String key) throws IOException {
byte[] bytes; byte[] bytes;
try(var in = TestRegionReassembler.class.getClassLoader().getResourceAsStream("region/r.9.-5.mca")){ try(var in = TestRegionReassembler.class.getClassLoader().getResourceAsStream(key)){
bytes = in.readAllBytes(); bytes = in.readAllBytes();
} }
try(var reasm = new RegionParser(Unpooled.wrappedBuffer(bytes), ByteBufAllocator.DEFAULT)) { try(var reasm = new RegionParser(Unpooled.wrappedBuffer(bytes), ByteBufAllocator.DEFAULT)) {
var result = reasm.writeReassembled(RegionFile.CompressType.NONE); var result = reasm.writeReassembled(RegionFile.CompressType.NONE);
try(var foc = FileChannel.open(Path.of("./test.mca"), StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
result.readBytes(foc, 0, result.readableBytes());
}
result.readerIndex(0);
try(var reader = new RegionFile(ByteBufAllocator.DEFAULT)) {
result = reasm.writeReassembled(RegionFile.CompressType.ZLIB);
}
} }
} }
} }
Binary file not shown.
+1 -1
View File
@@ -11,7 +11,7 @@ public record Backup(
long sizeTotal long sizeTotal
) { ) {
public Text toText(){ public Text toText(){
return Text.of("sizeTotal: "+((sizeTotal / 1024.0) / 1024.0) + "GiB, "+ return Text.of("sizeTotal: "+Math.round(((sizeTotal / 1024.0) / 1024.0) * 100) * 0.01 + "MiB, "+
(incremental ? "incremental" :"") + ", type: "+type+"." (incremental ? "incremental" :"") + ", type: "+type+"."
); );
} }
@@ -7,6 +7,7 @@ import com.mojang.brigadier.arguments.StringArgumentType;
import com.mojang.brigadier.context.CommandContext; import com.mojang.brigadier.context.CommandContext;
import com.mojang.brigadier.suggestion.Suggestions; import com.mojang.brigadier.suggestion.Suggestions;
import com.mojang.brigadier.suggestion.SuggestionsBuilder; import com.mojang.brigadier.suggestion.SuggestionsBuilder;
import io.ib67.sfcraft.bundler.BundleReader;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import net.minecraft.command.CommandRegistryAccess; import net.minecraft.command.CommandRegistryAccess;
import net.minecraft.server.command.CommandManager; import net.minecraft.server.command.CommandManager;
@@ -15,7 +16,10 @@ import net.minecraft.text.ClickEvent;
import net.minecraft.text.Text; import net.minecraft.text.Text;
import java.awt.*; import java.awt.*;
import java.io.IOException;
import java.lang.ref.WeakReference; import java.lang.ref.WeakReference;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
@@ -33,10 +37,10 @@ public class BackupCommands {
" -- Show this message", " -- Show this message",
" /backup list ", " /backup list ",
" -- List backups", " -- List backups",
" /backup restore <backupName>", " /backup restore <backupKey>",
" -- Restore the backup on next server restart.", " -- Restore the backup on next server restart.",
" Can be cancelled by /backup cancelrestore", " Can be cancelled by /backup cancelrestore",
" /backup delete <backupName>", " /backup delete <backupKey>",
" -- Delete a backup. This does not ask for a confirmation.", " -- Delete a backup. This does not ask for a confirmation.",
" /backup create <incremental: true/false> <subject / \"@all\">", " /backup create <incremental: true/false> <subject / \"@all\">",
" -- Create a backup.", " -- Create a backup.",
@@ -61,11 +65,11 @@ public class BackupCommands {
.then(literal("list").executes(this::cmdList)) .then(literal("list").executes(this::cmdList))
.then(literal("cancelrestore").executes(this::cmdCancelRestore)) .then(literal("cancelrestore").executes(this::cmdCancelRestore))
.then(literal("delete") .then(literal("delete")
.then(argument("backupName", StringArgumentType.greedyString()) .then(argument("backupKey", StringArgumentType.greedyString())
.suggests(this::suggestBackups) .suggests(this::suggestBackups)
.executes(this::cmdDelete))) .executes(this::cmdDelete)))
.then(literal("restore") .then(literal("restore")
.then(argument("backupName", StringArgumentType.greedyString()) .then(argument("backupKey", StringArgumentType.greedyString())
.suggests(this::suggestBackups) .suggests(this::suggestBackups)
.executes(this::cmdRestore))) .executes(this::cmdRestore)))
.then(literal("create") .then(literal("create")
@@ -128,19 +132,49 @@ public class BackupCommands {
s.sendMessage(Text.of("TIP: Use `/backup cancelrestore` to cancel that request.")); s.sendMessage(Text.of("TIP: Use `/backup cancelrestore` to cancel that request."));
return 0; return 0;
} }
var backupName = StringArgumentType.getString(context, "backupName"); var backupKey = StringArgumentType.getString(context, "backupKey");
var backup = mod.tracker.getTrackedBackups().get(backupKey);
if (backup == null) {
s.sendMessage(Text.of("Invalid backup " + backupKey + ". Note: Use backupKey instead of backupName!"));
return 0;
}
restoreIssued = () -> { restoreIssued = () -> {
//todo implementation log.warn("EXTRACTING BACKUP " + backup + " TO " + backup.from());
log.warn(" == STEP 1 == Make a backup for the destination.");
var destination = Path.of(backup.from());
var movedDst = destination.resolveSibling(destination.getFileName().toString()+"_bak_"+System.currentTimeMillis());
try{
Files.move(destination, movedDst);
log.info("{} has been moved to {}", destination, movedDst);
} catch (IOException e) {
log.error(e);
log.error("CANNOT MOVE "+destination+" TO "+movedDst);
log.error("BACKUP INTERRUPTED. Here are some tips helping you out:");
log.error(" -- A. Merge already moved files");
log.error(" Try this command in your server directory: (linux)");
log.error(" $ cp ./"+movedDst+"/* ./"+destination);
log.error(" -- B. Remove them all and use external unbundler tools");
log.error(" You may want to copy them elsewhere first (see kind A)");
log.error(" $ rm -r ./"+destination+" ./"+movedDst);
log.error(" Visit https://github.com/saltedfishclub/justbackup for the usage of bundler tools");
log.error(" ------- ERROR END -------");
return;
}
log.info(" == STEP 2 == Recovering backup "+backup);
mod.tracker.recoverBackup(backup, destination);
log.info("Backup has been recovered.");
}; };
s.sendMessage(Text.of("Backup restoration task has been scheduled!"));
s.sendMessage(Text.of("Restart your server to take changes."));
return Command.SINGLE_SUCCESS; return Command.SINGLE_SUCCESS;
} }
private int cmdDelete(CommandContext<ServerCommandSource> context) { private int cmdDelete(CommandContext<ServerCommandSource> context) {
var backupName = StringArgumentType.getString(context, "backupName"); var backupKey = StringArgumentType.getString(context, "backupKey");
var backup = mod.tracker.getTrackedBackups().get(backupName); var backup = mod.tracker.getTrackedBackups().get(backupKey);
var source = context.getSource(); var source = context.getSource();
if (backup == null) { if (backup == null) {
source.sendMessage(Text.of("Invalid backup. No backups are named '" + backupName + "'")); source.sendMessage(Text.of("Invalid backup. No backups are named '" + backupKey + "'"));
return 0; return 0;
} }
try { try {
@@ -194,6 +228,8 @@ public class BackupCommands {
for (String s : helpMessage) { for (String s : helpMessage) {
if (s.startsWith(" /")) { if (s.startsWith(" /")) {
source.sendMessage(Text.literal(s).withColor(Color.CYAN.getRGB())); source.sendMessage(Text.literal(s).withColor(Color.CYAN.getRGB()));
} else {
source.sendMessage(Text.of(s));
} }
} }
return Command.SINGLE_SUCCESS; return Command.SINGLE_SUCCESS;
@@ -33,19 +33,14 @@ public record BackupWorker(
if (server == null) { if (server == null) {
throw new IllegalStateException("MinecraftServer is not initialized yet, not doing backup"); throw new IllegalStateException("MinecraftServer is not initialized yet, not doing backup");
} }
int i = 0; server.saveAll(true, true, false);
while (!BACKUP_LOCK.compareAndSet(IOState.IDLE, IOState.BACKUP)) { while (!BACKUP_LOCK.compareAndSet(IOState.IDLE, IOState.BACKUP)) {
log.info("Server is saving world while performing backup! waiting..."); log.info("Server is saving world while performing backup! waiting...");
i++;
Thread.sleep(2000); Thread.sleep(2000);
} }
// performing backup is now true // performing backup is now true
var tmpBundle = tmpDir.resolve("backup_" + subject + "_" + System.currentTimeMillis() + ".swb.zst"); var tmpBundle = tmpDir.resolve("backup_" + subject.name() + "_" + System.currentTimeMillis() + ".swb.zst");
try { try {
if (i == 0) {
// not saved yet.
server.saveAll(true, true, false);
}
// tmp file path // tmp file path
log.info("Creating bundle {}", tmpBundle); log.info("Creating bundle {}", tmpBundle);
BundleWriter.createBundle(tmpBundle, subject.changedFiles(), config); BundleWriter.createBundle(tmpBundle, subject.changedFiles(), config);
@@ -1,8 +1,10 @@
package io.ib67.sfcraft; package io.ib67.sfcraft;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.extern.log4j.Log4j2;
import java.nio.file.*; import java.nio.file.*;
import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@@ -10,6 +12,7 @@ import java.util.function.BiConsumer;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
@Log4j2
public class FileWatcherThread extends Thread { public class FileWatcherThread extends Thread {
protected final Map<Path, String> watchSubject; protected final Map<Path, String> watchSubject;
protected final BiConsumer<String, Path> eventListener; protected final BiConsumer<String, Path> eventListener;
@@ -20,17 +23,24 @@ public class FileWatcherThread extends Thread {
) { ) {
this.watchSubject = watchSubject.entrySet() this.watchSubject = watchSubject.entrySet()
.stream() .stream()
.flatMap(it -> walkThrough(it.getValue()).map(r -> Map.entry(r, it.getKey()))) // todo support for nonexist dir
.flatMap(it -> walkThrough(it.getValue()).map(r -> Map.entry(r.toAbsolutePath(), it.getKey())))
.filter(it->Files.isDirectory(it.getKey()))
.collect(Collectors.toMap( .collect(Collectors.toMap(
Map.Entry::getKey, Map.Entry::getKey,
Map.Entry::getValue Map.Entry::getValue
)); ));
this.eventListener = eventListener; this.eventListener = eventListener;
setName("JustBackup/FileWatcher"); setName("JustBackup/FileWatcher");
setDaemon(true);
} }
@SneakyThrows @SneakyThrows
private static Stream<Path> walkThrough(Path path) { private static Stream<Path> walkThrough(Path path) {
if(Files.notExists(path)){
log.error("{} not found. Creating directory for it..", path);
Files.createDirectories(path);
}
return Files.walk(path); return Files.walk(path);
} }
@@ -38,12 +48,14 @@ public class FileWatcherThread extends Thread {
@SneakyThrows @SneakyThrows
public void run() { public void run() {
try (var watcher = FileSystems.getDefault().newWatchService()) { try (var watcher = FileSystems.getDefault().newWatchService()) {
var keyToParent = new HashMap<WatchKey, Path>();
for (Path value : watchSubject.keySet()) { for (Path value : watchSubject.keySet()) {
value.register( var key = value.register(
watcher, watcher,
StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_CREATE,
StandardWatchEventKinds.ENTRY_MODIFY StandardWatchEventKinds.ENTRY_MODIFY
); );
keyToParent.put(key, value);
} }
for (; ; ) { for (; ; ) {
var key = watcher.poll(10, TimeUnit.SECONDS); var key = watcher.poll(10, TimeUnit.SECONDS);
@@ -53,17 +65,25 @@ public class FileWatcherThread extends Thread {
var kind = ev.kind(); var kind = ev.kind();
if (kind == StandardWatchEventKinds.OVERFLOW) if (kind == StandardWatchEventKinds.OVERFLOW)
continue; continue;
var path = ((WatchEvent<Path>) ev).context(); var parent = keyToParent.get(key);
var subject = Objects.requireNonNull(watchSubject.get(path.getParent())); var subject = watchSubject.get(parent);
if(subject == null) {
log.warn("Ignoring changes in "+parent);
continue;
}
var path = parent.resolve(((WatchEvent<Path>) ev).context());
if (Files.isRegularFile(path)) { if (Files.isRegularFile(path)) {
eventListener.accept(subject, path.getParent()); eventListener.accept(subject, parent);
} else if (Files.isDirectory(path) && kind == StandardWatchEventKinds.ENTRY_CREATE) { } else if (Files.isDirectory(path) && kind == StandardWatchEventKinds.ENTRY_CREATE) {
path.register(watcher, StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_MODIFY); var nK = path.register(watcher, StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_MODIFY);
watchSubject.put(path, subject); watchSubject.put(path, subject);
keyToParent.put(nK, path);
} }
} }
key.reset(); key.reset();
} }
}catch (InterruptedException interruptedException){
log.info("FileWatcherThread is quitting!");
} }
} }
} }
@@ -30,13 +30,13 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors; import java.util.stream.Collectors;
public class JustBackupMod implements ModInitializer { public class JustBackupMod implements ModInitializer {
public static final String MOD_ID = "justbackup"; public static final String MOD_ID = "justbackup";
public static final Logger LOGGER = LoggerFactory.getLogger(MOD_ID); public static final Logger LOGGER = LoggerFactory.getLogger(MOD_ID);
protected final Map<String, BackupSubject> backupSubjects = new HashMap<>(); protected final Map<String, BackupSubject> backupSubjects = new HashMap<>();
protected int lastBackupServerTicks;
protected volatile MinecraftServer server; protected volatile MinecraftServer server;
protected volatile boolean suspend; protected volatile boolean suspend;
protected byte[] zstdDict; protected byte[] zstdDict;
@@ -63,7 +63,7 @@ public class JustBackupMod implements ModInitializer {
config.backupSubjects().forEach((k, v) -> { config.backupSubjects().forEach((k, v) -> {
var subjectPath = Path.of(v); var subjectPath = Path.of(v);
if (Files.notExists(subjectPath)) { if (Files.notExists(subjectPath)) {
LOGGER.error("Backup subject {} does not exist.", v); LOGGER.error("Backup _subject {} does not exist.", v);
return; return;
} }
backupSubjects.put(k, new BackupSubject(k, subjectPath, new ArrayList<>())); backupSubjects.put(k, new BackupSubject(k, subjectPath, new ArrayList<>()));
@@ -93,9 +93,11 @@ public class JustBackupMod implements ModInitializer {
watcherThread.start(); watcherThread.start();
} }
private void handleFileChanges(String subject, Path path) { private void handleFileChanges(String _subject, Path path) {
synchronized (backupSubjects) { synchronized (backupSubjects) {
backupSubjects.get(subject).changedFiles().add(path); var subject = backupSubjects.get(_subject);
if (subject != null)
subject.changedFiles().add(path); // check null
} }
} }
@@ -108,6 +110,12 @@ public class JustBackupMod implements ModInitializer {
backupAll(false); backupAll(false);
} }
scheduledBackupExecutor.scheduleAtFixedRate(() -> { scheduledBackupExecutor.scheduleAtFixedRate(() -> {
if (lastBackupServerTicks > 0 && lastBackupServerTicks == server.getTicks()) {
// the server is idling
LOGGER.debug("Server is idling, not backing up.");
return;
}
lastBackupServerTicks = server.getTicks();
if (!suspend) backupAll(config.incremental()).thenAccept(it -> { if (!suspend) backupAll(config.incremental()).thenAccept(it -> {
LOGGER.info("Backup success!"); LOGGER.info("Backup success!");
it.forEach((k, v) -> LOGGER.info(k + ": " + v)); it.forEach((k, v) -> LOGGER.info(k + ": " + v));
@@ -125,6 +133,7 @@ public class JustBackupMod implements ModInitializer {
var command = new BackupCommands(this); var command = new BackupCommands(this);
CommandRegistrationCallback.EVENT.register(command::registerCommand); CommandRegistrationCallback.EVENT.register(command::registerCommand);
ServerLifecycleEvents.SERVER_STOPPED.register(server -> { ServerLifecycleEvents.SERVER_STOPPED.register(server -> {
LOGGER.info("Shutting down the executor");
scheduledBackupExecutor.shutdown(); scheduledBackupExecutor.shutdown();
if (command.restoreIssued != null) { if (command.restoreIssued != null) {
command.restoreIssued.run(); command.restoreIssued.run();
@@ -134,6 +143,7 @@ public class JustBackupMod implements ModInitializer {
} }
public CompletableFuture<Map<String, Backup>> backupAll(boolean incremental) { public CompletableFuture<Map<String, Backup>> backupAll(boolean incremental) {
if (backupSubjects.isEmpty()) return CompletableFuture.completedFuture(Map.of());
var map = new HashMap<String, CompletableFuture<Backup>>(); var map = new HashMap<String, CompletableFuture<Backup>>();
server.getPlayerManager().broadcast( server.getPlayerManager().broadcast(
Text.literal(" BACKUP >> ").withColor(Color.RED.getRGB()).styled(it -> it.withBold(true)) Text.literal(" BACKUP >> ").withColor(Color.RED.getRGB()).styled(it -> it.withBold(true))
@@ -146,7 +156,11 @@ public class JustBackupMod implements ModInitializer {
return CompletableFuture.allOf(map.values().toArray(new CompletableFuture[0])) return CompletableFuture.allOf(map.values().toArray(new CompletableFuture[0]))
.thenApply(it -> { .thenApply(it -> {
var _map = new HashMap<String, Backup>(); var _map = new HashMap<String, Backup>();
map.forEach((k, v) -> _map.put(k, v.join())); map.forEach((k, v) -> {
var result = v.join();
LOGGER.info("Backup information of {}: {}", k, result);
_map.put(k, result);
});
return _map; return _map;
}); });
} }
@@ -154,15 +168,18 @@ public class JustBackupMod implements ModInitializer {
public CompletableFuture<Backup> issueBackup(String subjectName, boolean incremental) { public CompletableFuture<Backup> issueBackup(String subjectName, boolean incremental) {
var subject = backupSubjects.get(subjectName); var subject = backupSubjects.get(subjectName);
if (subject == null) if (subject == null)
return CompletableFuture.failedFuture(new IllegalArgumentException("Unknown backup subject: " + subjectName)); return CompletableFuture.failedFuture(new IllegalArgumentException("Unknown backup _subject: " + subjectName));
if (!incremental) { if (!incremental) {
try (var s = Files.walk(subject.root())) { try (var s = Files.walk(subject.root())) {
subject = new BackupSubject(subject.name(), subject.root(), s.filter(Files::isRegularFile).toList()); subject = new BackupSubject(subject.name(), subject.root(), s.filter(Files::isRegularFile)
.filter(it -> !it.getFileName().toString().equals("session.lock"))
.toList());
} catch (IOException e) { } catch (IOException e) {
return CompletableFuture.failedFuture(new IllegalArgumentException("Cannot perform full backup for " + subjectName, e)); return CompletableFuture.failedFuture(new IllegalArgumentException("Cannot perform full backup for " + subjectName, e));
} }
} }
var finalSubject = subject; var finalSubject = subject;
if (config.allowGunzip()) LOGGER.info("Chunk reassembler is enabled!");
var worker = new BackupWorker(server, tracker, Path.of(config.temporaryBackupDir()), var worker = new BackupWorker(server, tracker, Path.of(config.temporaryBackupDir()),
subject, c -> subject, c ->
c.allowGunzip(config.allowGunzip()) c.allowGunzip(config.allowGunzip())
@@ -26,14 +26,15 @@ public class LocalBackupStrategy implements BackupStrategy {
@Override @Override
@SneakyThrows @SneakyThrows
public Backup createBackup(String subject, Path from, Path pathToBundle) { public Backup createBackup(String subject, Path from, Path pathToBundle) {
var parent = backupParentRoot.resolve(subject);
var result = new Backup( var result = new Backup(
pathToBundle.getFileName().toString(), pathToBundle.getFileName().toString(),
pathToBundle.getFileName().toString(), Path.of(subject).resolve(pathToBundle.getFileName()).normalize().toString(),
"local", from.toString(), false, Files.size(pathToBundle)); "local", from.toString(), false, Files.size(pathToBundle));
var parent = backupParentRoot.resolve(subject);
if(Files.notExists(parent)) { if(Files.notExists(parent)) {
Files.createDirectories(parent); Files.createDirectories(parent);
} }
//todo new name
Files.move(pathToBundle, parent.resolve(pathToBundle.getFileName())); Files.move(pathToBundle, parent.resolve(pathToBundle.getFileName()));
return result; return result;
} }
@@ -3,7 +3,6 @@
"package": "io.ib67.sfcraft.mixin", "package": "io.ib67.sfcraft.mixin",
"compatibilityLevel": "JAVA_21", "compatibilityLevel": "JAVA_21",
"mixins": [ "mixins": [
"LevelStorageAccessor",
"MixinMinecraftServer", "MixinMinecraftServer",
"MixinRegionBasedStorage" "MixinRegionBasedStorage"
], ],