fix
This commit is contained in:
8 files changed
+9
-135
No files matched your search
+2
-3
@@ -64,15 +64,14 @@ dependencies {
|
||||
// Fabric API. This is technically optional, but you probably want it anyway.
|
||||
modImplementation "net.fabricmc.fabric-api:fabric-api:${project.fabric_version}"
|
||||
// https://mvnrepository.com/artifact/software.amazon.awssdk/s3
|
||||
api project(":bundler")
|
||||
shadow 'software.amazon.awssdk:s3:2.34.0'
|
||||
shadow 'com.github.luben:zstd-jni:1.5.7-4'
|
||||
shadow project(":bundler")
|
||||
}
|
||||
|
||||
configurations.implementation.extendsFrom(configurations.shadow)
|
||||
shadowJar {
|
||||
configurations = [project.configurations.shadow]
|
||||
exclude("META-INF")
|
||||
minimize()
|
||||
//relocate 'com.google', 'sfcraft.relocation.google'
|
||||
}
|
||||
|
||||
|
||||
@@ -10,9 +10,9 @@ dependencies {
|
||||
api 'com.github.luben:zstd-jni:1.5.7-4'
|
||||
compileOnly 'io.netty:netty-all:4.2.6.Final'
|
||||
api "io.ib67.kiwi:lang:1.2.3-SNAPSHOT"
|
||||
api 'io.netty:netty-all:4.2.6.Final'
|
||||
api 'it.unimi.dsi:fastutil:8.5.15'
|
||||
api 'org.lz4:lz4-java:1.8.0'
|
||||
compileOnly 'io.netty:netty-all:4.2.6.Final'
|
||||
compileOnly 'it.unimi.dsi:fastutil:8.5.15'
|
||||
compileOnly 'org.lz4:lz4-java:1.8.0'
|
||||
compileOnly 'org.lz4:lz4-java:1.8.0'
|
||||
testImplementation 'org.junit.jupiter:junit-jupiter:5.13.4'
|
||||
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
|
||||
|
||||
@@ -80,7 +80,7 @@ public class BundleWriter implements Closeable {
|
||||
futures.add(future);
|
||||
}
|
||||
return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).thenApply((v) ->
|
||||
Uni.from(futures::forEach).map(CompletableFuture::join).toList());
|
||||
futures.stream().map(CompletableFuture::join).toList());
|
||||
} else {
|
||||
return CompletableFuture.supplyAsync(() -> {
|
||||
var singleResult = parentOfBundle.resolve("result.zst");
|
||||
|
||||
Binary file not shown.
@@ -7,7 +7,6 @@ import io.ib67.sfcraft.config.StorageOption;
|
||||
import io.ib67.sfcraft.strategy.BackupStrategy;
|
||||
import io.ib67.sfcraft.strategy.BackupTracker;
|
||||
import io.ib67.sfcraft.strategy.LocalBackupStrategy;
|
||||
import io.ib67.sfcraft.strategy.S3BackupStrategy;
|
||||
import io.netty.buffer.ByteBufAllocator;
|
||||
import lombok.SneakyThrows;
|
||||
import net.fabricmc.api.ModInitializer;
|
||||
@@ -180,7 +179,7 @@ public class JustBackupMod implements ModInitializer {
|
||||
private BackupStrategy createStrategy() {
|
||||
return switch (config.option()) {
|
||||
case StorageOption.Local local -> new LocalBackupStrategy(local);
|
||||
case StorageOption.S3 s3 -> new S3BackupStrategy(s3, Path.of(config.temporaryBackupDir()));
|
||||
case StorageOption.S3 s3 -> throw new IllegalStateException();
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
package io.ib67.sfcraft.mixin;
|
||||
|
||||
import net.minecraft.server.MinecraftServer;
|
||||
import net.minecraft.world.level.storage.LevelStorage;
|
||||
import org.spongepowered.asm.mixin.Mixin;
|
||||
import org.spongepowered.asm.mixin.gen.Accessor;
|
||||
|
||||
@Mixin(MinecraftServer.class)
|
||||
public interface LevelStorageAccessor {
|
||||
@Accessor
|
||||
LevelStorage.Session getSession();
|
||||
}
|
||||
@@ -20,8 +20,8 @@ public class BackupTracker implements BackupStrategy {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Backup createBackup(Path from, Path pathToBundle) {
|
||||
var result = underlying.createBackup(from, pathToBundle);
|
||||
public Backup createBackup(String subject, Path from, Path pathToBundle) {
|
||||
var result = underlying.createBackup(subject, from, pathToBundle);
|
||||
trackedBackups.put(result.backupKey(), result);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
package io.ib67.sfcraft.strategy;
|
||||
|
||||
import io.ib67.sfcraft.Backup;
|
||||
import io.ib67.sfcraft.bundler.BundleReader;
|
||||
import io.ib67.sfcraft.config.StorageOption;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
@Log4j2
|
||||
public class S3BackupStrategy implements BackupStrategy {
|
||||
protected final AtomicBoolean uploading = new AtomicBoolean(false);
|
||||
protected final StorageOption.S3 option;
|
||||
protected final S3Client s3;
|
||||
protected final Path temporaryDownloadPath;
|
||||
|
||||
@SneakyThrows
|
||||
public S3BackupStrategy(StorageOption.S3 option, Path tempDownloadPath) {
|
||||
this.option = option;
|
||||
var credential = AwsBasicCredentials.create(option.accessKey(), option.secretKey());
|
||||
this.s3 = S3Client.builder()
|
||||
.region(Region.of(option.region()))
|
||||
.forcePathStyle(option.enforcePathStyle())
|
||||
.endpointOverride(URI.create(option.endpoint()))
|
||||
.credentialsProvider(StaticCredentialsProvider.create(credential))
|
||||
.build();
|
||||
log.info("Testing S3 credentials");
|
||||
// throws exception if the bucket neither inaccessible nor not exist
|
||||
s3.headBucket(b -> b.bucket(option.bucket()));
|
||||
this.temporaryDownloadPath = Objects.requireNonNull(tempDownloadPath);
|
||||
if (Files.notExists(temporaryDownloadPath))
|
||||
Files.createDirectories(temporaryDownloadPath);
|
||||
}
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public Backup createBackup(Path from, Path pathToBundle) {
|
||||
if (!uploading.compareAndSet(false, true)) {
|
||||
throw new IllegalStateException("Last upload is not done yet! keep waiting...");
|
||||
}
|
||||
int i = 0;
|
||||
do {
|
||||
try {
|
||||
var object = option.prefix() + "/" + pathToBundle.getFileName().toString();
|
||||
s3.putObject(o -> o.bucket(option.bucket())
|
||||
.contentType("application/octet-stream")
|
||||
.ifNoneMatch("*") // to avoid content overriding
|
||||
.key(object).build(), pathToBundle);
|
||||
return new Backup(pathToBundle.getFileName().toString(),
|
||||
object, "s3",
|
||||
from.toString(),
|
||||
false,
|
||||
Files.size(pathToBundle));
|
||||
} catch (IOException e) {
|
||||
if (i >= option.retryAmount()) {
|
||||
throw new IllegalStateException("Cannot upload " + pathToBundle + " to S3");
|
||||
}
|
||||
var nextTry = Math.pow(2, i++);
|
||||
log.error("Error uploading S3. trying in {}s later. ({}/{})", nextTry, i, option.retryAmount(), e);
|
||||
Thread.sleep((long) (nextTry * 1000));
|
||||
} catch (Exception e) {
|
||||
throw new IllegalStateException("Unrecoverable exception occurred when uploading " + pathToBundle, e);
|
||||
} finally {
|
||||
uploading.set(false);
|
||||
}
|
||||
} while (true);
|
||||
}
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public void recoverBackup(Backup backup, Path restorePath) {
|
||||
var resp = s3.getObject(b -> b.bucket(option.bucket()).key(backup.backupKey()));
|
||||
var target = temporaryDownloadPath.resolve("s3_" + System.currentTimeMillis());
|
||||
try (var fs = Files.newOutputStream(target, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
|
||||
resp.transferTo(fs);
|
||||
BundleReader.builder().build().extract(target, restorePath);
|
||||
} finally {
|
||||
Files.deleteIfExists(target);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public void deleteBackup(Backup backup) {
|
||||
s3.deleteObject(b -> b.bucket(option.bucket()).key(backup.backupKey()).build());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isAvailable() {
|
||||
if (uploading.get()) {
|
||||
log.warn("Last S3 Backup is still uploading!");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
s3.close();
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user