fix fix fix
This commit is contained in:
1 parent
394a4d986d
commit
48bf045825
16 files changed
+172
-59
No files matched your search
+3
-1
@@ -67,13 +67,15 @@ dependencies {
|
|||||||
shadow project(":bundler")
|
shadow project(":bundler")
|
||||||
shadow 'software.amazon.awssdk:s3:2.34.0'
|
shadow 'software.amazon.awssdk:s3:2.34.0'
|
||||||
}
|
}
|
||||||
|
|
||||||
configurations.implementation.extendsFrom(configurations.shadow)
|
configurations.implementation.extendsFrom(configurations.shadow)
|
||||||
shadowJar {
|
shadowJar {
|
||||||
configurations = [project.configurations.shadow]
|
configurations = [project.configurations.shadow]
|
||||||
exclude("META-INF")
|
exclude("META-INF")
|
||||||
dependencies {
|
dependencies {
|
||||||
exclude(dependency("software.amazon.awssdk:aws-xml-protocol:.*"))
|
exclude(dependency("software.amazon.awssdk:aws-xml-protocol:.*"))
|
||||||
|
exclude(dependency("org.lz4:lz4-java:.*"))
|
||||||
|
exclude(dependency("it.unimi.dsi:fastutil:.*"))
|
||||||
|
exclude(dependency("io.netty:netty-all:.*"))
|
||||||
}
|
}
|
||||||
minimize()
|
minimize()
|
||||||
//relocate 'com.google', 'sfcraft.relocation.google'
|
//relocate 'com.google', 'sfcraft.relocation.google'
|
||||||
|
|||||||
@@ -8,18 +8,19 @@ plugins {
|
|||||||
|
|
||||||
dependencies {
|
dependencies {
|
||||||
api 'com.github.luben:zstd-jni:1.5.7-4'
|
api 'com.github.luben:zstd-jni:1.5.7-4'
|
||||||
compileOnly 'io.netty:netty-all:4.2.6.Final'
|
testCompileOnly 'io.netty:netty-all:4.2.6.Final'
|
||||||
api "io.ib67.kiwi:lang:1.2.3-SNAPSHOT"
|
api "io.ib67.kiwi:lang:1.2.3-SNAPSHOT"
|
||||||
compileOnly 'io.netty:netty-all:4.2.6.Final'
|
implementation 'io.netty:netty-all:4.2.6.Final'
|
||||||
compileOnly 'it.unimi.dsi:fastutil:8.5.15'
|
implementation 'it.unimi.dsi:fastutil:8.5.15'
|
||||||
compileOnly 'org.lz4:lz4-java:1.8.0'
|
implementation 'org.lz4:lz4-java:1.8.0'
|
||||||
compileOnly 'org.lz4:lz4-java:1.8.0'
|
|
||||||
testImplementation 'org.junit.jupiter:junit-jupiter:5.13.4'
|
testImplementation 'org.junit.jupiter:junit-jupiter:5.13.4'
|
||||||
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
|
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
|
||||||
}
|
}
|
||||||
|
|
||||||
shadowJar {
|
configurations {
|
||||||
minimize()
|
testRuntimeOnly {
|
||||||
|
extendsFrom compileOnly
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
test {
|
test {
|
||||||
|
|||||||
@@ -62,12 +62,18 @@ public class BundleReader {
|
|||||||
entryData.writeBytes(buffer, 0, read);
|
entryData.writeBytes(buffer, 0, read);
|
||||||
}
|
}
|
||||||
try (var reassembler = new RegionParser(entryData, allocator)) {
|
try (var reassembler = new RegionParser(entryData, allocator)) {
|
||||||
var result = reassembler.writeReassembled(RegionFile.CompressType.GZIP);
|
var result = reassembler.writeReassembled(RegionFile.CompressType.ZLIB);
|
||||||
try (var fc = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
|
try (var fc = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
|
||||||
result.readBytes(fc, 0, result.readableBytes());
|
result.readBytes(fc, 0, result.readableBytes());
|
||||||
} finally {
|
} finally {
|
||||||
result.release();
|
result.release();
|
||||||
}
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
try (var ch = FileChannel.open(Path.of("dump.mca"), StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
|
||||||
|
entryData.readerIndex(0);
|
||||||
|
entryData.readBytes(ch, 0, entryData.readableBytes());
|
||||||
|
}
|
||||||
|
throw new IOException("Error while reassembling " + entry.name(), e);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
writePlain(path, entry, in);
|
writePlain(path, entry, in);
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ public class BundleWriter implements Closeable {
|
|||||||
var futures = new ArrayList<CompletableFuture<Path>>();
|
var futures = new ArrayList<CompletableFuture<Path>>();
|
||||||
for (int i = 0; i < parted.size(); i++) {
|
for (int i = 0; i < parted.size(); i++) {
|
||||||
var pathList = parted.get(i);
|
var pathList = parted.get(i);
|
||||||
var output = parentOfBundle.resolve("Worker_" + System.currentTimeMillis() + "_" + i + ".jbp.zst");
|
var output = parentOfBundle.resolve("Worker_" + System.currentTimeMillis() + "_" + i + ".swb.zst");
|
||||||
var future = CompletableFuture.supplyAsync(() -> createBundleWorker(pathList, output, config), executor);
|
var future = CompletableFuture.supplyAsync(() -> createBundleWorker(pathList, output, config), executor);
|
||||||
futures.add(future);
|
futures.add(future);
|
||||||
}
|
}
|
||||||
@@ -142,7 +142,7 @@ public class BundleWriter implements Closeable {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public void write(Path path, boolean suggestGunzip) throws IOException {
|
public void write(Path path, boolean suggestGunzip) throws IOException {
|
||||||
if (suggestGunzip) {
|
if (suggestGunzip && allowReassemble) {
|
||||||
//todo size limit
|
//todo size limit
|
||||||
writeReassembleInMem(path);
|
writeReassembleInMem(path);
|
||||||
return;
|
return;
|
||||||
@@ -158,22 +158,26 @@ public class BundleWriter implements Closeable {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if(verbose) System.out.println("Reassembling " + path);
|
if(verbose) System.out.println("Reassembling " + path);
|
||||||
var uncompressed = allocator.buffer();
|
var finalResult = allocator.buffer();
|
||||||
var rPath = relativeRoot.toAbsolutePath().relativize(path.toAbsolutePath());
|
var rPath = relativeRoot.toAbsolutePath().relativize(path.toAbsolutePath());
|
||||||
try (var out = new EntryOutputStream(new ByteBufOutputStream(uncompressed));
|
try (var out = new EntryOutputStream(new ByteBufOutputStream(finalResult));
|
||||||
var rf = new RegionParser(path, allocator)) {
|
var rf = new RegionParser(path, allocator)) {
|
||||||
|
// uncompressed result
|
||||||
var result = rf.writeReassembled(RegionFile.CompressType.NONE);
|
var result = rf.writeReassembled(RegionFile.CompressType.NONE);
|
||||||
var length = result.readableBytes();
|
var length = result.readableBytes();
|
||||||
out.writeEntryHeader(rPath.toString(), BundleEntry.ATTR_REASSEMBLE, length, System.currentTimeMillis());
|
out.writeEntryHeader(rPath.toString(), BundleEntry.ATTR_REASSEMBLE, length, System.currentTimeMillis());
|
||||||
result.readBytes(out, length);
|
result.readBytes(out, length);
|
||||||
result.release();
|
result.release();
|
||||||
uncompressed.readBytes(outputStream, uncompressed.readableBytes());
|
// all done
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
System.err.println("Failed to parse region " + path + ": " + e);
|
System.err.println("Failed to parse region " + path + ": " + e);
|
||||||
|
finalResult.release();
|
||||||
writePlain(path);
|
writePlain(path);
|
||||||
} finally {
|
return;
|
||||||
uncompressed.release();
|
|
||||||
}
|
}
|
||||||
|
finalResult.readBytes(outputStream, finalResult.readableBytes());
|
||||||
|
finalResult.release();
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void writePlain(Path path) throws IOException {
|
private void writePlain(Path path) throws IOException {
|
||||||
@@ -186,6 +190,9 @@ public class BundleWriter implements Closeable {
|
|||||||
while ((read = fs.read(buffer)) > 0) {
|
while ((read = fs.read(buffer)) > 0) {
|
||||||
outputStream.write(buffer, 0, read);
|
outputStream.write(buffer, 0, read);
|
||||||
}
|
}
|
||||||
|
}catch(IOException e){
|
||||||
|
System.err.println("Error while bundling "+path);
|
||||||
|
throw e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ public class Main {
|
|||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
var opts = ArgOpts.builder()
|
var opts = ArgOpts.builder()
|
||||||
.args(args).description("Tools for .jbp.zst format bundle").programName("jpack").build();
|
.args(args).description("Tools for .swb.zst format bundle").programName("jpack").build();
|
||||||
var in = opts.string("in", "Input. Can be directory or jpack bundle", null);
|
var in = opts.string("in", "Input. Can be directory or jpack bundle", null);
|
||||||
var out = opts.string("out", "Output. Can be jpack bundle or directory", null);
|
var out = opts.string("out", "Output. Can be jpack bundle or directory", null);
|
||||||
var help = opts.bool("help", false);
|
var help = opts.bool("help", false);
|
||||||
@@ -81,9 +81,9 @@ public class Main {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (in.endsWith(".jbp.zst")) {
|
if (in.endsWith(".swb.zst")) {
|
||||||
BundleReader.builder().build().extract(inPath, outPath);
|
BundleReader.builder().build().extract(inPath, outPath);
|
||||||
} else if (out.endsWith(".jbp.zst")) {
|
} else if (out.endsWith(".swb.zst")) {
|
||||||
UnaryOperator<BundleWriter.BundleWriterBuilder> cfg = it -> it
|
UnaryOperator<BundleWriter.BundleWriterBuilder> cfg = it -> it
|
||||||
.allowGunzip(allowGunzip)
|
.allowGunzip(allowGunzip)
|
||||||
.relativeRoot(inPath)
|
.relativeRoot(inPath)
|
||||||
|
|||||||
@@ -89,11 +89,14 @@ public class RegionFile implements Closeable {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public ByteBuf writeOutput() {
|
public ByteBuf writeOutput() {
|
||||||
var sect = allocator.buffer(8192);
|
var sect = allocator.buffer(4096);
|
||||||
|
sect.setZero(0, 4096);
|
||||||
|
sect.writerIndex(4096);
|
||||||
var compositeByteBuf = allocator.compositeBuffer(2 + regionData.size());
|
var compositeByteBuf = allocator.compositeBuffer(2 + regionData.size());
|
||||||
var beginSector = 2;
|
var beginSector = 2;
|
||||||
compositeByteBuf.addComponent(true, sect);
|
compositeByteBuf.addComponent(true, sect);
|
||||||
timestamp.retain();
|
timestamp.retain();
|
||||||
|
timestamp.readerIndex(0);
|
||||||
compositeByteBuf.addComponent(true, timestamp);
|
compositeByteBuf.addComponent(true, timestamp);
|
||||||
for (var entry : regionData.short2ObjectEntrySet()) {
|
for (var entry : regionData.short2ObjectEntrySet()) {
|
||||||
var chunkKey = entry.getShortKey();
|
var chunkKey = entry.getShortKey();
|
||||||
@@ -102,12 +105,21 @@ public class RegionFile implements Closeable {
|
|||||||
var chunk = entry.getValue();
|
var chunk = entry.getValue();
|
||||||
var size = chunk.readableBytes();
|
var size = chunk.readableBytes();
|
||||||
var sizeSectors = Math.ceilDiv(size, 4096);
|
var sizeSectors = Math.ceilDiv(size, 4096);
|
||||||
|
var zeros = 4096 - (size % 4096);
|
||||||
|
for (int i = 0; i < zeros; i++) {
|
||||||
|
chunk.writeByte(0);
|
||||||
|
}
|
||||||
if (sizeSectors > 255) {
|
if (sizeSectors > 255) {
|
||||||
throw new IllegalArgumentException("MCC not supported.");
|
throw new IllegalArgumentException("MCC not supported.");
|
||||||
}
|
}
|
||||||
var indexEntry = Integer.reverseBytes(beginSector) << 1 | (sizeSectors & 0xFF);
|
if(sizeSectors < 0) {
|
||||||
// sect = int[1024]
|
throw new IllegalArgumentException("Invalid sizeSectors: "+sizeSectors);
|
||||||
sect.setInt((x + 32 * z) * 4, indexEntry);
|
}
|
||||||
|
var begin = (x + 32 * z) * 4;
|
||||||
|
sect.setByte(begin, (beginSector >>> 16) & 0xFF);
|
||||||
|
sect.setByte(begin+1, (beginSector >>> 8) & 0xFF);
|
||||||
|
sect.setByte(begin+2, beginSector & 0xFF);
|
||||||
|
sect.setByte(begin+3, sizeSectors & 0xFF);
|
||||||
chunk.retain(); // retain a local reference
|
chunk.retain(); // retain a local reference
|
||||||
compositeByteBuf.addComponent(true, chunk);
|
compositeByteBuf.addComponent(true, chunk);
|
||||||
beginSector += sizeSectors;
|
beginSector += sizeSectors;
|
||||||
|
|||||||
@@ -22,12 +22,12 @@ public class RegionParser implements Closeable {
|
|||||||
this.allocator = allocator;
|
this.allocator = allocator;
|
||||||
var size = (int) Files.size(file);
|
var size = (int) Files.size(file);
|
||||||
if (size < 8192) throw new IOException("The region file " + file + " has been truncated");
|
if (size < 8192) throw new IOException("The region file " + file + " has been truncated");
|
||||||
if(size > 8388608 * 2) throw new IOException("The region file " + file + " is too large"); //todo bug
|
if (size > 8388608 * 2) throw new IOException("The region file " + file + " is too large"); //todo bug
|
||||||
source = allocator.buffer(size);
|
source = allocator.buffer(size);
|
||||||
|
|
||||||
try (var fc = FileChannel.open(file, StandardOpenOption.READ)) {
|
try (var fc = FileChannel.open(file, StandardOpenOption.READ)) {
|
||||||
var toRead = size;
|
var toRead = size;
|
||||||
while(toRead > 0){
|
while (toRead > 0) {
|
||||||
toRead -= source.writeBytes(fc, 0L, (int) size);
|
toRead -= source.writeBytes(fc, 0L, (int) size);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -66,6 +66,7 @@ public class RegionParser implements Closeable {
|
|||||||
if (compressType < 0) {
|
if (compressType < 0) {
|
||||||
// mcc todo
|
// mcc todo
|
||||||
}
|
}
|
||||||
|
if (sectorLen == 0) return compressType;
|
||||||
buf.writeBytes(source, sectorLen - 1);
|
buf.writeBytes(source, sectorLen - 1);
|
||||||
return compressType;
|
return compressType;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,16 +7,32 @@ import io.netty.buffer.Unpooled;
|
|||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.nio.channels.FileChannel;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.StandardOpenOption;
|
||||||
|
|
||||||
public class TestRegionReassembler {
|
public class TestRegionReassembler {
|
||||||
@Test
|
@Test
|
||||||
public void testReassembler() throws IOException {
|
public void testReassembler() throws IOException {
|
||||||
|
testMethod("region/r.9.-5.mca");
|
||||||
|
testMethod("region/r.-1.-1.mca");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void testMethod(String key) throws IOException {
|
||||||
byte[] bytes;
|
byte[] bytes;
|
||||||
try(var in = TestRegionReassembler.class.getClassLoader().getResourceAsStream("region/r.9.-5.mca")){
|
try(var in = TestRegionReassembler.class.getClassLoader().getResourceAsStream(key)){
|
||||||
bytes = in.readAllBytes();
|
bytes = in.readAllBytes();
|
||||||
}
|
}
|
||||||
try(var reasm = new RegionParser(Unpooled.wrappedBuffer(bytes), ByteBufAllocator.DEFAULT)) {
|
try(var reasm = new RegionParser(Unpooled.wrappedBuffer(bytes), ByteBufAllocator.DEFAULT)) {
|
||||||
var result = reasm.writeReassembled(RegionFile.CompressType.NONE);
|
var result = reasm.writeReassembled(RegionFile.CompressType.NONE);
|
||||||
|
try(var foc = FileChannel.open(Path.of("./test.mca"), StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
|
||||||
|
result.readBytes(foc, 0, result.readableBytes());
|
||||||
|
}
|
||||||
|
result.readerIndex(0);
|
||||||
|
try(var reader = new RegionFile(ByteBufAllocator.DEFAULT)) {
|
||||||
|
result = reasm.writeReassembled(RegionFile.CompressType.ZLIB);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Binary file not shown.
@@ -11,7 +11,7 @@ public record Backup(
|
|||||||
long sizeTotal
|
long sizeTotal
|
||||||
) {
|
) {
|
||||||
public Text toText(){
|
public Text toText(){
|
||||||
return Text.of("sizeTotal: "+((sizeTotal / 1024.0) / 1024.0) + "GiB, "+
|
return Text.of("sizeTotal: "+Math.round(((sizeTotal / 1024.0) / 1024.0) * 100) * 0.01 + "MiB, "+
|
||||||
(incremental ? "incremental" :"") + ", type: "+type+"."
|
(incremental ? "incremental" :"") + ", type: "+type+"."
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import com.mojang.brigadier.arguments.StringArgumentType;
|
|||||||
import com.mojang.brigadier.context.CommandContext;
|
import com.mojang.brigadier.context.CommandContext;
|
||||||
import com.mojang.brigadier.suggestion.Suggestions;
|
import com.mojang.brigadier.suggestion.Suggestions;
|
||||||
import com.mojang.brigadier.suggestion.SuggestionsBuilder;
|
import com.mojang.brigadier.suggestion.SuggestionsBuilder;
|
||||||
|
import io.ib67.sfcraft.bundler.BundleReader;
|
||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
import net.minecraft.command.CommandRegistryAccess;
|
import net.minecraft.command.CommandRegistryAccess;
|
||||||
import net.minecraft.server.command.CommandManager;
|
import net.minecraft.server.command.CommandManager;
|
||||||
@@ -15,7 +16,10 @@ import net.minecraft.text.ClickEvent;
|
|||||||
import net.minecraft.text.Text;
|
import net.minecraft.text.Text;
|
||||||
|
|
||||||
import java.awt.*;
|
import java.awt.*;
|
||||||
|
import java.io.IOException;
|
||||||
import java.lang.ref.WeakReference;
|
import java.lang.ref.WeakReference;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.concurrent.CompletableFuture;
|
import java.util.concurrent.CompletableFuture;
|
||||||
@@ -33,10 +37,10 @@ public class BackupCommands {
|
|||||||
" -- Show this message",
|
" -- Show this message",
|
||||||
" /backup list ",
|
" /backup list ",
|
||||||
" -- List backups",
|
" -- List backups",
|
||||||
" /backup restore <backupName>",
|
" /backup restore <backupKey>",
|
||||||
" -- Restore the backup on next server restart.",
|
" -- Restore the backup on next server restart.",
|
||||||
" Can be cancelled by /backup cancelrestore",
|
" Can be cancelled by /backup cancelrestore",
|
||||||
" /backup delete <backupName>",
|
" /backup delete <backupKey>",
|
||||||
" -- Delete a backup. This does not ask for a confirmation.",
|
" -- Delete a backup. This does not ask for a confirmation.",
|
||||||
" /backup create <incremental: true/false> <subject / \"@all\">",
|
" /backup create <incremental: true/false> <subject / \"@all\">",
|
||||||
" -- Create a backup.",
|
" -- Create a backup.",
|
||||||
@@ -61,11 +65,11 @@ public class BackupCommands {
|
|||||||
.then(literal("list").executes(this::cmdList))
|
.then(literal("list").executes(this::cmdList))
|
||||||
.then(literal("cancelrestore").executes(this::cmdCancelRestore))
|
.then(literal("cancelrestore").executes(this::cmdCancelRestore))
|
||||||
.then(literal("delete")
|
.then(literal("delete")
|
||||||
.then(argument("backupName", StringArgumentType.greedyString())
|
.then(argument("backupKey", StringArgumentType.greedyString())
|
||||||
.suggests(this::suggestBackups)
|
.suggests(this::suggestBackups)
|
||||||
.executes(this::cmdDelete)))
|
.executes(this::cmdDelete)))
|
||||||
.then(literal("restore")
|
.then(literal("restore")
|
||||||
.then(argument("backupName", StringArgumentType.greedyString())
|
.then(argument("backupKey", StringArgumentType.greedyString())
|
||||||
.suggests(this::suggestBackups)
|
.suggests(this::suggestBackups)
|
||||||
.executes(this::cmdRestore)))
|
.executes(this::cmdRestore)))
|
||||||
.then(literal("create")
|
.then(literal("create")
|
||||||
@@ -128,19 +132,49 @@ public class BackupCommands {
|
|||||||
s.sendMessage(Text.of("TIP: Use `/backup cancelrestore` to cancel that request."));
|
s.sendMessage(Text.of("TIP: Use `/backup cancelrestore` to cancel that request."));
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
var backupName = StringArgumentType.getString(context, "backupName");
|
var backupKey = StringArgumentType.getString(context, "backupKey");
|
||||||
|
var backup = mod.tracker.getTrackedBackups().get(backupKey);
|
||||||
|
if (backup == null) {
|
||||||
|
s.sendMessage(Text.of("Invalid backup " + backupKey + ". Note: Use backupKey instead of backupName!"));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
restoreIssued = () -> {
|
restoreIssued = () -> {
|
||||||
//todo implementation
|
log.warn("EXTRACTING BACKUP " + backup + " TO " + backup.from());
|
||||||
|
log.warn(" == STEP 1 == Make a backup for the destination.");
|
||||||
|
var destination = Path.of(backup.from());
|
||||||
|
var movedDst = destination.resolveSibling(destination.getFileName().toString()+"_bak_"+System.currentTimeMillis());
|
||||||
|
try{
|
||||||
|
Files.move(destination, movedDst);
|
||||||
|
log.info("{} has been moved to {}", destination, movedDst);
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error(e);
|
||||||
|
log.error("CANNOT MOVE "+destination+" TO "+movedDst);
|
||||||
|
log.error("BACKUP INTERRUPTED. Here are some tips helping you out:");
|
||||||
|
log.error(" -- A. Merge already moved files");
|
||||||
|
log.error(" Try this command in your server directory: (linux)");
|
||||||
|
log.error(" $ cp ./"+movedDst+"/* ./"+destination);
|
||||||
|
log.error(" -- B. Remove them all and use external unbundler tools");
|
||||||
|
log.error(" You may want to copy them elsewhere first (see kind A)");
|
||||||
|
log.error(" $ rm -r ./"+destination+" ./"+movedDst);
|
||||||
|
log.error(" Visit https://github.com/saltedfishclub/justbackup for the usage of bundler tools");
|
||||||
|
log.error(" ------- ERROR END -------");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
log.info(" == STEP 2 == Recovering backup "+backup);
|
||||||
|
mod.tracker.recoverBackup(backup, destination);
|
||||||
|
log.info("Backup has been recovered.");
|
||||||
};
|
};
|
||||||
|
s.sendMessage(Text.of("Backup restoration task has been scheduled!"));
|
||||||
|
s.sendMessage(Text.of("Restart your server to take changes."));
|
||||||
return Command.SINGLE_SUCCESS;
|
return Command.SINGLE_SUCCESS;
|
||||||
}
|
}
|
||||||
|
|
||||||
private int cmdDelete(CommandContext<ServerCommandSource> context) {
|
private int cmdDelete(CommandContext<ServerCommandSource> context) {
|
||||||
var backupName = StringArgumentType.getString(context, "backupName");
|
var backupKey = StringArgumentType.getString(context, "backupKey");
|
||||||
var backup = mod.tracker.getTrackedBackups().get(backupName);
|
var backup = mod.tracker.getTrackedBackups().get(backupKey);
|
||||||
var source = context.getSource();
|
var source = context.getSource();
|
||||||
if (backup == null) {
|
if (backup == null) {
|
||||||
source.sendMessage(Text.of("Invalid backup. No backups are named '" + backupName + "'"));
|
source.sendMessage(Text.of("Invalid backup. No backups are named '" + backupKey + "'"));
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
@@ -194,6 +228,8 @@ public class BackupCommands {
|
|||||||
for (String s : helpMessage) {
|
for (String s : helpMessage) {
|
||||||
if (s.startsWith(" /")) {
|
if (s.startsWith(" /")) {
|
||||||
source.sendMessage(Text.literal(s).withColor(Color.CYAN.getRGB()));
|
source.sendMessage(Text.literal(s).withColor(Color.CYAN.getRGB()));
|
||||||
|
} else {
|
||||||
|
source.sendMessage(Text.of(s));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return Command.SINGLE_SUCCESS;
|
return Command.SINGLE_SUCCESS;
|
||||||
|
|||||||
@@ -33,19 +33,14 @@ public record BackupWorker(
|
|||||||
if (server == null) {
|
if (server == null) {
|
||||||
throw new IllegalStateException("MinecraftServer is not initialized yet, not doing backup");
|
throw new IllegalStateException("MinecraftServer is not initialized yet, not doing backup");
|
||||||
}
|
}
|
||||||
int i = 0;
|
server.saveAll(true, true, false);
|
||||||
while (!BACKUP_LOCK.compareAndSet(IOState.IDLE, IOState.BACKUP)) {
|
while (!BACKUP_LOCK.compareAndSet(IOState.IDLE, IOState.BACKUP)) {
|
||||||
log.info("Server is saving world while performing backup! waiting...");
|
log.info("Server is saving world while performing backup! waiting...");
|
||||||
i++;
|
|
||||||
Thread.sleep(2000);
|
Thread.sleep(2000);
|
||||||
}
|
}
|
||||||
// performing backup is now true
|
// performing backup is now true
|
||||||
var tmpBundle = tmpDir.resolve("backup_" + subject + "_" + System.currentTimeMillis() + ".swb.zst");
|
var tmpBundle = tmpDir.resolve("backup_" + subject.name() + "_" + System.currentTimeMillis() + ".swb.zst");
|
||||||
try {
|
try {
|
||||||
if (i == 0) {
|
|
||||||
// not saved yet.
|
|
||||||
server.saveAll(true, true, false);
|
|
||||||
}
|
|
||||||
// tmp file path
|
// tmp file path
|
||||||
log.info("Creating bundle {}", tmpBundle);
|
log.info("Creating bundle {}", tmpBundle);
|
||||||
BundleWriter.createBundle(tmpBundle, subject.changedFiles(), config);
|
BundleWriter.createBundle(tmpBundle, subject.changedFiles(), config);
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
package io.ib67.sfcraft;
|
package io.ib67.sfcraft;
|
||||||
|
|
||||||
import lombok.SneakyThrows;
|
import lombok.SneakyThrows;
|
||||||
|
import lombok.extern.log4j.Log4j2;
|
||||||
|
|
||||||
import java.nio.file.*;
|
import java.nio.file.*;
|
||||||
|
import java.util.HashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
@@ -10,6 +12,7 @@ import java.util.function.BiConsumer;
|
|||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
import java.util.stream.Stream;
|
import java.util.stream.Stream;
|
||||||
|
|
||||||
|
@Log4j2
|
||||||
public class FileWatcherThread extends Thread {
|
public class FileWatcherThread extends Thread {
|
||||||
protected final Map<Path, String> watchSubject;
|
protected final Map<Path, String> watchSubject;
|
||||||
protected final BiConsumer<String, Path> eventListener;
|
protected final BiConsumer<String, Path> eventListener;
|
||||||
@@ -20,17 +23,24 @@ public class FileWatcherThread extends Thread {
|
|||||||
) {
|
) {
|
||||||
this.watchSubject = watchSubject.entrySet()
|
this.watchSubject = watchSubject.entrySet()
|
||||||
.stream()
|
.stream()
|
||||||
.flatMap(it -> walkThrough(it.getValue()).map(r -> Map.entry(r, it.getKey())))
|
// todo support for nonexist dir
|
||||||
|
.flatMap(it -> walkThrough(it.getValue()).map(r -> Map.entry(r.toAbsolutePath(), it.getKey())))
|
||||||
|
.filter(it->Files.isDirectory(it.getKey()))
|
||||||
.collect(Collectors.toMap(
|
.collect(Collectors.toMap(
|
||||||
Map.Entry::getKey,
|
Map.Entry::getKey,
|
||||||
Map.Entry::getValue
|
Map.Entry::getValue
|
||||||
));
|
));
|
||||||
this.eventListener = eventListener;
|
this.eventListener = eventListener;
|
||||||
setName("JustBackup/FileWatcher");
|
setName("JustBackup/FileWatcher");
|
||||||
|
setDaemon(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
private static Stream<Path> walkThrough(Path path) {
|
private static Stream<Path> walkThrough(Path path) {
|
||||||
|
if(Files.notExists(path)){
|
||||||
|
log.error("{} not found. Creating directory for it..", path);
|
||||||
|
Files.createDirectories(path);
|
||||||
|
}
|
||||||
return Files.walk(path);
|
return Files.walk(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -38,12 +48,14 @@ public class FileWatcherThread extends Thread {
|
|||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public void run() {
|
public void run() {
|
||||||
try (var watcher = FileSystems.getDefault().newWatchService()) {
|
try (var watcher = FileSystems.getDefault().newWatchService()) {
|
||||||
|
var keyToParent = new HashMap<WatchKey, Path>();
|
||||||
for (Path value : watchSubject.keySet()) {
|
for (Path value : watchSubject.keySet()) {
|
||||||
value.register(
|
var key = value.register(
|
||||||
watcher,
|
watcher,
|
||||||
StandardWatchEventKinds.ENTRY_CREATE,
|
StandardWatchEventKinds.ENTRY_CREATE,
|
||||||
StandardWatchEventKinds.ENTRY_MODIFY
|
StandardWatchEventKinds.ENTRY_MODIFY
|
||||||
);
|
);
|
||||||
|
keyToParent.put(key, value);
|
||||||
}
|
}
|
||||||
for (; ; ) {
|
for (; ; ) {
|
||||||
var key = watcher.poll(10, TimeUnit.SECONDS);
|
var key = watcher.poll(10, TimeUnit.SECONDS);
|
||||||
@@ -53,17 +65,25 @@ public class FileWatcherThread extends Thread {
|
|||||||
var kind = ev.kind();
|
var kind = ev.kind();
|
||||||
if (kind == StandardWatchEventKinds.OVERFLOW)
|
if (kind == StandardWatchEventKinds.OVERFLOW)
|
||||||
continue;
|
continue;
|
||||||
var path = ((WatchEvent<Path>) ev).context();
|
var parent = keyToParent.get(key);
|
||||||
var subject = Objects.requireNonNull(watchSubject.get(path.getParent()));
|
var subject = watchSubject.get(parent);
|
||||||
|
if(subject == null) {
|
||||||
|
log.warn("Ignoring changes in "+parent);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
var path = parent.resolve(((WatchEvent<Path>) ev).context());
|
||||||
if (Files.isRegularFile(path)) {
|
if (Files.isRegularFile(path)) {
|
||||||
eventListener.accept(subject, path.getParent());
|
eventListener.accept(subject, parent);
|
||||||
} else if (Files.isDirectory(path) && kind == StandardWatchEventKinds.ENTRY_CREATE) {
|
} else if (Files.isDirectory(path) && kind == StandardWatchEventKinds.ENTRY_CREATE) {
|
||||||
path.register(watcher, StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_MODIFY);
|
var nK = path.register(watcher, StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_MODIFY);
|
||||||
watchSubject.put(path, subject);
|
watchSubject.put(path, subject);
|
||||||
|
keyToParent.put(nK, path);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
key.reset();
|
key.reset();
|
||||||
}
|
}
|
||||||
|
}catch (InterruptedException interruptedException){
|
||||||
|
log.info("FileWatcherThread is quitting!");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -30,13 +30,13 @@ import java.util.concurrent.CompletableFuture;
|
|||||||
import java.util.concurrent.Executors;
|
import java.util.concurrent.Executors;
|
||||||
import java.util.concurrent.ScheduledExecutorService;
|
import java.util.concurrent.ScheduledExecutorService;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
public class JustBackupMod implements ModInitializer {
|
public class JustBackupMod implements ModInitializer {
|
||||||
public static final String MOD_ID = "justbackup";
|
public static final String MOD_ID = "justbackup";
|
||||||
public static final Logger LOGGER = LoggerFactory.getLogger(MOD_ID);
|
public static final Logger LOGGER = LoggerFactory.getLogger(MOD_ID);
|
||||||
protected final Map<String, BackupSubject> backupSubjects = new HashMap<>();
|
protected final Map<String, BackupSubject> backupSubjects = new HashMap<>();
|
||||||
|
protected int lastBackupServerTicks;
|
||||||
protected volatile MinecraftServer server;
|
protected volatile MinecraftServer server;
|
||||||
protected volatile boolean suspend;
|
protected volatile boolean suspend;
|
||||||
protected byte[] zstdDict;
|
protected byte[] zstdDict;
|
||||||
@@ -63,7 +63,7 @@ public class JustBackupMod implements ModInitializer {
|
|||||||
config.backupSubjects().forEach((k, v) -> {
|
config.backupSubjects().forEach((k, v) -> {
|
||||||
var subjectPath = Path.of(v);
|
var subjectPath = Path.of(v);
|
||||||
if (Files.notExists(subjectPath)) {
|
if (Files.notExists(subjectPath)) {
|
||||||
LOGGER.error("Backup subject {} does not exist.", v);
|
LOGGER.error("Backup _subject {} does not exist.", v);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
backupSubjects.put(k, new BackupSubject(k, subjectPath, new ArrayList<>()));
|
backupSubjects.put(k, new BackupSubject(k, subjectPath, new ArrayList<>()));
|
||||||
@@ -93,9 +93,11 @@ public class JustBackupMod implements ModInitializer {
|
|||||||
watcherThread.start();
|
watcherThread.start();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void handleFileChanges(String subject, Path path) {
|
private void handleFileChanges(String _subject, Path path) {
|
||||||
synchronized (backupSubjects) {
|
synchronized (backupSubjects) {
|
||||||
backupSubjects.get(subject).changedFiles().add(path);
|
var subject = backupSubjects.get(_subject);
|
||||||
|
if (subject != null)
|
||||||
|
subject.changedFiles().add(path); // check null
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -108,6 +110,12 @@ public class JustBackupMod implements ModInitializer {
|
|||||||
backupAll(false);
|
backupAll(false);
|
||||||
}
|
}
|
||||||
scheduledBackupExecutor.scheduleAtFixedRate(() -> {
|
scheduledBackupExecutor.scheduleAtFixedRate(() -> {
|
||||||
|
if (lastBackupServerTicks > 0 && lastBackupServerTicks == server.getTicks()) {
|
||||||
|
// the server is idling
|
||||||
|
LOGGER.debug("Server is idling, not backing up.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
lastBackupServerTicks = server.getTicks();
|
||||||
if (!suspend) backupAll(config.incremental()).thenAccept(it -> {
|
if (!suspend) backupAll(config.incremental()).thenAccept(it -> {
|
||||||
LOGGER.info("Backup success!");
|
LOGGER.info("Backup success!");
|
||||||
it.forEach((k, v) -> LOGGER.info(k + ": " + v));
|
it.forEach((k, v) -> LOGGER.info(k + ": " + v));
|
||||||
@@ -125,6 +133,7 @@ public class JustBackupMod implements ModInitializer {
|
|||||||
var command = new BackupCommands(this);
|
var command = new BackupCommands(this);
|
||||||
CommandRegistrationCallback.EVENT.register(command::registerCommand);
|
CommandRegistrationCallback.EVENT.register(command::registerCommand);
|
||||||
ServerLifecycleEvents.SERVER_STOPPED.register(server -> {
|
ServerLifecycleEvents.SERVER_STOPPED.register(server -> {
|
||||||
|
LOGGER.info("Shutting down the executor");
|
||||||
scheduledBackupExecutor.shutdown();
|
scheduledBackupExecutor.shutdown();
|
||||||
if (command.restoreIssued != null) {
|
if (command.restoreIssued != null) {
|
||||||
command.restoreIssued.run();
|
command.restoreIssued.run();
|
||||||
@@ -134,6 +143,7 @@ public class JustBackupMod implements ModInitializer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public CompletableFuture<Map<String, Backup>> backupAll(boolean incremental) {
|
public CompletableFuture<Map<String, Backup>> backupAll(boolean incremental) {
|
||||||
|
if (backupSubjects.isEmpty()) return CompletableFuture.completedFuture(Map.of());
|
||||||
var map = new HashMap<String, CompletableFuture<Backup>>();
|
var map = new HashMap<String, CompletableFuture<Backup>>();
|
||||||
server.getPlayerManager().broadcast(
|
server.getPlayerManager().broadcast(
|
||||||
Text.literal(" BACKUP >> ").withColor(Color.RED.getRGB()).styled(it -> it.withBold(true))
|
Text.literal(" BACKUP >> ").withColor(Color.RED.getRGB()).styled(it -> it.withBold(true))
|
||||||
@@ -146,7 +156,11 @@ public class JustBackupMod implements ModInitializer {
|
|||||||
return CompletableFuture.allOf(map.values().toArray(new CompletableFuture[0]))
|
return CompletableFuture.allOf(map.values().toArray(new CompletableFuture[0]))
|
||||||
.thenApply(it -> {
|
.thenApply(it -> {
|
||||||
var _map = new HashMap<String, Backup>();
|
var _map = new HashMap<String, Backup>();
|
||||||
map.forEach((k, v) -> _map.put(k, v.join()));
|
map.forEach((k, v) -> {
|
||||||
|
var result = v.join();
|
||||||
|
LOGGER.info("Backup information of {}: {}", k, result);
|
||||||
|
_map.put(k, result);
|
||||||
|
});
|
||||||
return _map;
|
return _map;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -154,15 +168,18 @@ public class JustBackupMod implements ModInitializer {
|
|||||||
public CompletableFuture<Backup> issueBackup(String subjectName, boolean incremental) {
|
public CompletableFuture<Backup> issueBackup(String subjectName, boolean incremental) {
|
||||||
var subject = backupSubjects.get(subjectName);
|
var subject = backupSubjects.get(subjectName);
|
||||||
if (subject == null)
|
if (subject == null)
|
||||||
return CompletableFuture.failedFuture(new IllegalArgumentException("Unknown backup subject: " + subjectName));
|
return CompletableFuture.failedFuture(new IllegalArgumentException("Unknown backup _subject: " + subjectName));
|
||||||
if (!incremental) {
|
if (!incremental) {
|
||||||
try (var s = Files.walk(subject.root())) {
|
try (var s = Files.walk(subject.root())) {
|
||||||
subject = new BackupSubject(subject.name(), subject.root(), s.filter(Files::isRegularFile).toList());
|
subject = new BackupSubject(subject.name(), subject.root(), s.filter(Files::isRegularFile)
|
||||||
|
.filter(it -> !it.getFileName().toString().equals("session.lock"))
|
||||||
|
.toList());
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
return CompletableFuture.failedFuture(new IllegalArgumentException("Cannot perform full backup for " + subjectName, e));
|
return CompletableFuture.failedFuture(new IllegalArgumentException("Cannot perform full backup for " + subjectName, e));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
var finalSubject = subject;
|
var finalSubject = subject;
|
||||||
|
if (config.allowGunzip()) LOGGER.info("Chunk reassembler is enabled!");
|
||||||
var worker = new BackupWorker(server, tracker, Path.of(config.temporaryBackupDir()),
|
var worker = new BackupWorker(server, tracker, Path.of(config.temporaryBackupDir()),
|
||||||
subject, c ->
|
subject, c ->
|
||||||
c.allowGunzip(config.allowGunzip())
|
c.allowGunzip(config.allowGunzip())
|
||||||
|
|||||||
@@ -26,14 +26,15 @@ public class LocalBackupStrategy implements BackupStrategy {
|
|||||||
@Override
|
@Override
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public Backup createBackup(String subject, Path from, Path pathToBundle) {
|
public Backup createBackup(String subject, Path from, Path pathToBundle) {
|
||||||
|
var parent = backupParentRoot.resolve(subject);
|
||||||
var result = new Backup(
|
var result = new Backup(
|
||||||
pathToBundle.getFileName().toString(),
|
pathToBundle.getFileName().toString(),
|
||||||
pathToBundle.getFileName().toString(),
|
Path.of(subject).resolve(pathToBundle.getFileName()).normalize().toString(),
|
||||||
"local", from.toString(), false, Files.size(pathToBundle));
|
"local", from.toString(), false, Files.size(pathToBundle));
|
||||||
var parent = backupParentRoot.resolve(subject);
|
|
||||||
if(Files.notExists(parent)) {
|
if(Files.notExists(parent)) {
|
||||||
Files.createDirectories(parent);
|
Files.createDirectories(parent);
|
||||||
}
|
}
|
||||||
|
//todo new name
|
||||||
Files.move(pathToBundle, parent.resolve(pathToBundle.getFileName()));
|
Files.move(pathToBundle, parent.resolve(pathToBundle.getFileName()));
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,6 @@
|
|||||||
"package": "io.ib67.sfcraft.mixin",
|
"package": "io.ib67.sfcraft.mixin",
|
||||||
"compatibilityLevel": "JAVA_21",
|
"compatibilityLevel": "JAVA_21",
|
||||||
"mixins": [
|
"mixins": [
|
||||||
"LevelStorageAccessor",
|
|
||||||
"MixinMinecraftServer",
|
"MixinMinecraftServer",
|
||||||
"MixinRegionBasedStorage"
|
"MixinRegionBasedStorage"
|
||||||
],
|
],
|
||||||
|
|||||||
Reference in new issue
Block a user