fix fix fix
use amazon s3
This commit is contained in:
1 parent
fb42be8155
commit
048451543b
7 files changed
+79
-35
No files matched your search
@@ -7,8 +7,10 @@ import net.minecraft.text.ClickEvent;
|
|||||||
import net.minecraft.text.Text;
|
import net.minecraft.text.Text;
|
||||||
|
|
||||||
import java.awt.*;
|
import java.awt.*;
|
||||||
|
import java.io.IOException;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
|
import java.util.Comparator;
|
||||||
|
|
||||||
public class BackupCommands {
|
public class BackupCommands {
|
||||||
protected final JustBackupMod mod;
|
protected final JustBackupMod mod;
|
||||||
@@ -81,12 +83,24 @@ public class BackupCommands {
|
|||||||
s.sendMessage(Text.literal("Please restart your server to take changes"));
|
s.sendMessage(Text.literal("Please restart your server to take changes"));
|
||||||
restoreIssued = () -> {
|
restoreIssued = () -> {
|
||||||
System.out.println("Recovering backup... ");
|
System.out.println("Recovering backup... ");
|
||||||
try{
|
try {
|
||||||
var toOverride = Path.of(backup.from());
|
var toOverride = Path.of(backup.from());
|
||||||
var old = Path.of(backup.from()+"_old");
|
var old = Path.of(backup.from() + "_old");
|
||||||
|
if(Files.exists(old)){
|
||||||
|
try (var f = Files.walk(old)) {
|
||||||
|
f.sorted(Comparator.reverseOrder())
|
||||||
|
.forEach(it -> {
|
||||||
|
try {
|
||||||
|
Files.deleteIfExists(it);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
Files.deleteIfExists(old);
|
Files.deleteIfExists(old);
|
||||||
Files.move(toOverride, old);
|
Files.move(toOverride, old);
|
||||||
System.out.println("Moved your old save to "+old);
|
System.out.println("Moved your old save to " + old);
|
||||||
mod.tracker.recoverBackup(backup, toOverride);
|
mod.tracker.recoverBackup(backup, toOverride);
|
||||||
System.out.println("Recover successfully!");
|
System.out.println("Recover successfully!");
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
|||||||
@@ -145,12 +145,19 @@ public class JustBackupMod implements ModInitializer {
|
|||||||
.then(literal("create").executes(commandHandler::cmdIssueBackup))
|
.then(literal("create").executes(commandHandler::cmdIssueBackup))
|
||||||
.then(literal("suspend").executes(commandHandler::cmdSuspend))
|
.then(literal("suspend").executes(commandHandler::cmdSuspend))
|
||||||
);
|
);
|
||||||
|
ServerLifecycleEvents.SERVER_STOPPED.register(server -> {
|
||||||
|
scheduledBackupExecutor.shutdown();
|
||||||
|
if (commandHandler.restoreIssued != null) {
|
||||||
|
commandHandler.restoreIssued.run();
|
||||||
|
}
|
||||||
|
tracker.close();
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private BackupStrategy createStrategy() {
|
private BackupStrategy createStrategy() {
|
||||||
return switch (config.option()) {
|
return switch (config.option()) {
|
||||||
case StorageOption.Local local -> new LocalBackupStrategy(local);
|
case StorageOption.Local local -> new LocalBackupStrategy(local);
|
||||||
case StorageOption.S3 s3 -> new S3BackupStrategy(s3);
|
case StorageOption.S3 s3 -> new S3BackupStrategy(s3, Path.of(config.temporaryBackupDir()));
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ public class SerialBundlerReader implements Closeable {
|
|||||||
|
|
||||||
protected final EntryInputStream metadataIn;
|
protected final EntryInputStream metadataIn;
|
||||||
protected final byte[] buffer = new byte[4096];
|
protected final byte[] buffer = new byte[4096];
|
||||||
|
protected final RandomAccessFile raf;
|
||||||
protected boolean entryRead;
|
protected boolean entryRead;
|
||||||
protected int entries;
|
protected int entries;
|
||||||
|
|
||||||
@@ -28,7 +29,7 @@ public class SerialBundlerReader implements Closeable {
|
|||||||
// }
|
// }
|
||||||
|
|
||||||
public SerialBundlerReader(Path path) throws IOException {
|
public SerialBundlerReader(Path path) throws IOException {
|
||||||
var raf = new RandomAccessFile(path.toFile(), "r");
|
raf = new RandomAccessFile(path.toFile(), "r");
|
||||||
var totalLen = Files.size(path);
|
var totalLen = Files.size(path);
|
||||||
var magic = raf.readInt();
|
var magic = raf.readInt();
|
||||||
var compressed = magic == Bundle.COMPRESSED_MAGIC;
|
var compressed = magic == Bundle.COMPRESSED_MAGIC;
|
||||||
@@ -96,5 +97,6 @@ public class SerialBundlerReader implements Closeable {
|
|||||||
public void close() throws IOException {
|
public void close() throws IOException {
|
||||||
in.close();
|
in.close();
|
||||||
metadataIn.close();
|
metadataIn.close();
|
||||||
|
raf.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -18,4 +18,6 @@ public interface BackupStrategy {
|
|||||||
void deleteBackup(Backup backup);
|
void deleteBackup(Backup backup);
|
||||||
|
|
||||||
boolean isAvailable();
|
boolean isAvailable();
|
||||||
|
|
||||||
|
void close();
|
||||||
}
|
}
|
||||||
@@ -41,4 +41,9 @@ public class BackupTracker implements BackupStrategy {
|
|||||||
public boolean isAvailable() {
|
public boolean isAvailable() {
|
||||||
return underlying.isAvailable();
|
return underlying.isAvailable();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
underlying.close();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -80,4 +80,9 @@ public class LocalBackupStrategy implements BackupStrategy {
|
|||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -1,48 +1,46 @@
|
|||||||
package io.ib67.sfcraft.strategy;
|
package io.ib67.sfcraft.strategy;
|
||||||
|
|
||||||
import io.ib67.sfcraft.Backup;
|
import io.ib67.sfcraft.Backup;
|
||||||
import io.ib67.sfcraft.WorldDir;
|
|
||||||
import io.ib67.sfcraft.bundler.Bundle;
|
import io.ib67.sfcraft.bundler.Bundle;
|
||||||
import io.ib67.sfcraft.config.StorageOption;
|
import io.ib67.sfcraft.config.StorageOption;
|
||||||
import io.minio.BucketExistsArgs;
|
|
||||||
import io.minio.MinioClient;
|
|
||||||
import io.minio.RemoveObjectArgs;
|
|
||||||
import io.minio.UploadObjectArgs;
|
|
||||||
import lombok.RequiredArgsConstructor;
|
|
||||||
import lombok.SneakyThrows;
|
import lombok.SneakyThrows;
|
||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
|
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||||
|
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||||
|
import software.amazon.awssdk.regions.Region;
|
||||||
|
import software.amazon.awssdk.services.s3.S3Client;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.MalformedURLException;
|
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URL;
|
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.nio.file.StandardOpenOption;
|
||||||
|
import java.util.Objects;
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
|
||||||
@Log4j2
|
@Log4j2
|
||||||
public class S3BackupStrategy implements BackupStrategy {
|
public class S3BackupStrategy implements BackupStrategy {
|
||||||
protected final StorageOption.S3 option;
|
|
||||||
protected final MinioClient minio;
|
|
||||||
protected final AtomicBoolean uploading = new AtomicBoolean(false);
|
protected final AtomicBoolean uploading = new AtomicBoolean(false);
|
||||||
|
protected final StorageOption.S3 option;
|
||||||
|
protected final S3Client s3;
|
||||||
|
protected final Path temporaryDownloadPath;
|
||||||
|
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public S3BackupStrategy(StorageOption.S3 option) {
|
public S3BackupStrategy(StorageOption.S3 option, Path tempDownloadPath) {
|
||||||
this.option = option;
|
this.option = option;
|
||||||
this.minio = MinioClient.builder()
|
var credential = AwsBasicCredentials.create(option.accessKey(), option.secretKey());
|
||||||
.endpoint(URI.create(option.endpoint()).toURL())
|
this.s3 = S3Client.builder()
|
||||||
.credentials(option.accessKey(), option.secretKey())
|
.region(Region.of(option.region()))
|
||||||
.region(option.region())
|
.forcePathStyle(option.enforcePathStyle())
|
||||||
|
.endpointOverride(URI.create(option.endpoint()))
|
||||||
|
.credentialsProvider(StaticCredentialsProvider.create(credential))
|
||||||
.build();
|
.build();
|
||||||
if (option.enforcePathStyle()) {
|
|
||||||
minio.disableVirtualStyleEndpoint();
|
|
||||||
} else {
|
|
||||||
minio.enableVirtualStyleEndpoint();
|
|
||||||
}
|
|
||||||
log.info("Testing S3 credentials");
|
log.info("Testing S3 credentials");
|
||||||
var exists = minio.bucketExists(BucketExistsArgs.builder().bucket(option.bucket()).build());
|
// throws exception if the bucket neither inaccessible nor not exist
|
||||||
if (!exists) throw new IllegalStateException("Bucket '" + option.bucket() + "' does not exist");
|
s3.headBucket(b -> b.bucket(option.bucket()));
|
||||||
|
this.temporaryDownloadPath = Objects.requireNonNull(tempDownloadPath);
|
||||||
|
if (Files.notExists(temporaryDownloadPath))
|
||||||
|
Files.createDirectories(temporaryDownloadPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -55,11 +53,10 @@ public class S3BackupStrategy implements BackupStrategy {
|
|||||||
do {
|
do {
|
||||||
try {
|
try {
|
||||||
var object = option.prefix() + "/" + pathToBundle.getFileName().toString();
|
var object = option.prefix() + "/" + pathToBundle.getFileName().toString();
|
||||||
minio.uploadObject(UploadObjectArgs.builder()
|
s3.putObject(o -> o.bucket(option.bucket())
|
||||||
.contentType("application/octet-stream")
|
.contentType("application/octet-stream")
|
||||||
.object(object)
|
.ifNoneMatch("*") // to avoid content overriding
|
||||||
.filename(pathToBundle.toString())
|
.key(object).build(), pathToBundle);
|
||||||
.build());
|
|
||||||
//todo also calculate sha?
|
//todo also calculate sha?
|
||||||
return new Backup(pathToBundle.getFileName().toString(),
|
return new Backup(pathToBundle.getFileName().toString(),
|
||||||
object, "s3",
|
object, "s3",
|
||||||
@@ -82,15 +79,22 @@ public class S3BackupStrategy implements BackupStrategy {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@SneakyThrows
|
||||||
public void recoverBackup(Backup backup, Path restorePath) {
|
public void recoverBackup(Backup backup, Path restorePath) {
|
||||||
//todo http range
|
var resp = s3.getObject(b -> b.bucket(option.bucket()).key(backup.backupKey()));
|
||||||
|
var target = temporaryDownloadPath.resolve("s3_" + System.currentTimeMillis());
|
||||||
|
try (var fs = Files.newOutputStream(target, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
|
||||||
|
resp.transferTo(fs);
|
||||||
|
Bundle.unbundleFile(target, restorePath);
|
||||||
|
} finally {
|
||||||
|
Files.deleteIfExists(target);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public void deleteBackup(Backup backup) {
|
public void deleteBackup(Backup backup) {
|
||||||
minio.removeObject(RemoveObjectArgs.builder()
|
s3.deleteObject(b -> b.bucket(option.bucket()).key(backup.backupKey()).build());
|
||||||
.bucket(backup.backupKey()).build());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -101,4 +105,9 @@ public class S3BackupStrategy implements BackupStrategy {
|
|||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
s3.close();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user