diff --git a/build.gradle b/build.gradle
index de4b57b..2fa2fa0 100644
--- a/build.gradle
+++ b/build.gradle
@@ -68,9 +68,7 @@ dependencies {
// Fabric API. This is technically optional, but you probably want it anyway.
implementation "net.fabricmc.fabric-api:fabric-api:${project.fabric_version}"
- // https://mvnrepository.com/artifact/software.amazon.awssdk/s3
shadow project(":bundler")
- shadow 'software.amazon.awssdk:s3:2.34.0'
testImplementation 'org.junit.jupiter:junit-jupiter:5.13.4'
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
@@ -84,7 +82,6 @@ shadowJar {
configurations = [project.configurations.shadow]
exclude("META-INF")
dependencies {
- exclude(dependency("software.amazon.awssdk:aws-xml-protocol:.*"))
exclude(dependency("org.lz4:lz4-java:.*"))
exclude(dependency("it.unimi.dsi:fastutil:.*"))
exclude(dependency("io.netty:netty-all:.*"))
diff --git a/src/main/java/io/ib67/sfcraft/s3/AwsV4Signer.java b/src/main/java/io/ib67/sfcraft/s3/AwsV4Signer.java
new file mode 100644
index 0000000..0530250
--- /dev/null
+++ b/src/main/java/io/ib67/sfcraft/s3/AwsV4Signer.java
@@ -0,0 +1,81 @@
+package io.ib67.sfcraft.s3;
+
+import javax.crypto.Mac;
+import javax.crypto.spec.SecretKeySpec;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.security.MessageDigest;
+import java.util.HexFormat;
+import java.util.SortedMap;
+import java.util.StringJoiner;
+
+/**
+ * Minimal AWS Signature Version 4 signer for S3.
+ *
+ * @see
+ * Authenticating Requests: Using the Authorization Header (AWS Signature Version 4)
+ */
+final class AwsV4Signer {
+ // SHA-256 of a zero-length payload
+ static final String EMPTY_PAYLOAD_SHA256 =
+ "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+ private static final HexFormat HEX = HexFormat.of();
+
+ private AwsV4Signer() {
+ }
+
+ /**
+ * @param encodedPath already URI-encoded absolute path, exactly as sent on the wire
+ * @param headers headers to sign; keys must be lowercase, values trimmed.
+ * Must include host, x-amz-date and x-amz-content-sha256.
+ */
+ static String authorization(String method, String encodedPath, String canonicalQuery,
+ SortedMap headers, String payloadHash,
+ String amzDate, String region, String service,
+ String accessKey, String secretKey) {
+ var canonicalHeaders = new StringBuilder();
+ var signedHeaders = new StringJoiner(";");
+ for (var header : headers.entrySet()) {
+ canonicalHeaders.append(header.getKey()).append(':').append(header.getValue()).append('\n');
+ signedHeaders.add(header.getKey());
+ }
+ var canonicalRequest = method + '\n'
+ + encodedPath + '\n'
+ + canonicalQuery + '\n'
+ + canonicalHeaders + '\n'
+ + signedHeaders + '\n'
+ + payloadHash;
+ var dateStamp = amzDate.substring(0, 8);
+ var scope = dateStamp + "/" + region + "/" + service + "/aws4_request";
+ var stringToSign = "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n"
+ + sha256Hex(canonicalRequest.getBytes(StandardCharsets.UTF_8));
+ var signingKey = hmac(hmac(hmac(hmac(
+ ("AWS4" + secretKey).getBytes(StandardCharsets.UTF_8), dateStamp), region), service), "aws4_request");
+ return "AWS4-HMAC-SHA256 Credential=" + accessKey + "/" + scope
+ + ",SignedHeaders=" + signedHeaders
+ + ",Signature=" + HEX.formatHex(hmac(signingKey, stringToSign));
+ }
+
+ static String sha256Hex(byte[] data) {
+ return HEX.formatHex(sha256().digest(data));
+ }
+
+ static MessageDigest sha256() {
+ try {
+ return MessageDigest.getInstance("SHA-256");
+ } catch (GeneralSecurityException e) {
+ throw new IllegalStateException("JVM lacks SHA-256", e);
+ }
+ }
+
+ private static byte[] hmac(byte[] key, String data) {
+ try {
+ var mac = Mac.getInstance("HmacSHA256");
+ mac.init(new SecretKeySpec(key, "HmacSHA256"));
+ return mac.doFinal(data.getBytes(StandardCharsets.UTF_8));
+ } catch (GeneralSecurityException e) {
+ throw new IllegalStateException("JVM lacks HmacSHA256", e);
+ }
+ }
+}
diff --git a/src/main/java/io/ib67/sfcraft/s3/S3Exception.java b/src/main/java/io/ib67/sfcraft/s3/S3Exception.java
new file mode 100644
index 0000000..36eb7c6
--- /dev/null
+++ b/src/main/java/io/ib67/sfcraft/s3/S3Exception.java
@@ -0,0 +1,21 @@
+package io.ib67.sfcraft.s3;
+
+import java.io.IOException;
+
+/**
+ * A non-2xx response from the S3 endpoint. Network failures surface as plain
+ * {@link IOException}s instead.
+ */
+public final class S3Exception extends IOException {
+ private final int statusCode;
+
+ public S3Exception(String request, int statusCode, String errorBody) {
+ super(request + " -> HTTP " + statusCode + (errorBody == null || errorBody.isBlank()
+ ? "" : ": " + errorBody.strip()));
+ this.statusCode = statusCode;
+ }
+
+ public int statusCode() {
+ return statusCode;
+ }
+}
diff --git a/src/main/java/io/ib67/sfcraft/s3/SimpleS3Client.java b/src/main/java/io/ib67/sfcraft/s3/SimpleS3Client.java
new file mode 100644
index 0000000..0f3be9b
--- /dev/null
+++ b/src/main/java/io/ib67/sfcraft/s3/SimpleS3Client.java
@@ -0,0 +1,170 @@
+package io.ib67.sfcraft.s3;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URLEncoder;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.HexFormat;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.TreeMap;
+
+/**
+ * A tiny synchronous S3 client on top of {@link HttpClient}, covering only what
+ * this mod needs: headBucket, putObject, getObject and deleteObject. Requests
+ * are authenticated with AWS Signature Version 4 and work against any
+ * S3-compatible endpoint, in both path-style and virtual-hosted style.
+ */
+public final class SimpleS3Client implements AutoCloseable {
+ private static final DateTimeFormatter AMZ_DATE =
+ DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'", Locale.ROOT).withZone(ZoneOffset.UTC);
+
+ private final HttpClient http;
+ private final String scheme;
+ // authority as HttpClient will put it into the Host header: default ports stripped
+ private final String hostPort;
+ private final String region;
+ private final String accessKey;
+ private final String secretKey;
+ private final boolean pathStyle;
+
+ public SimpleS3Client(String endpoint, String region, String accessKey, String secretKey, boolean pathStyle) {
+ var uri = URI.create(endpoint);
+ this.scheme = Objects.requireNonNull(uri.getScheme(), "endpoint has no scheme: " + endpoint);
+ Objects.requireNonNull(uri.getHost(), "endpoint has no host: " + endpoint);
+ var defaultPort = "https".equals(scheme) ? 443 : 80;
+ this.hostPort = uri.getPort() == -1 || uri.getPort() == defaultPort
+ ? uri.getHost()
+ : uri.getHost() + ":" + uri.getPort();
+ this.region = Objects.requireNonNull(region);
+ this.accessKey = Objects.requireNonNull(accessKey);
+ this.secretKey = Objects.requireNonNull(secretKey);
+ this.pathStyle = pathStyle;
+ this.http = HttpClient.newBuilder()
+ .version(HttpClient.Version.HTTP_1_1)
+ .followRedirects(HttpClient.Redirect.NEVER)
+ .connectTimeout(Duration.ofSeconds(10))
+ .build();
+ }
+
+ /**
+ * Throws {@link S3Exception} if the bucket does not exist or the credentials
+ * cannot access it.
+ */
+ public void headBucket(String bucket) throws IOException, InterruptedException {
+ var target = target(bucket, null);
+ var request = signed("HEAD", target, AwsV4Signer.EMPTY_PAYLOAD_SHA256)
+ .HEAD().build();
+ var response = http.send(request, HttpResponse.BodyHandlers.discarding());
+ ensureSuccess("HEAD " + target.uri(), response.statusCode(), null);
+ }
+
+ public void putObject(String bucket, String key, Path file, String contentType)
+ throws IOException, InterruptedException {
+ var target = target(bucket, key);
+ var request = signed("PUT", target, sha256Hex(file))
+ .header("content-type", contentType)
+ .PUT(HttpRequest.BodyPublishers.ofFile(file))
+ .build();
+ var response = http.send(request, HttpResponse.BodyHandlers.ofString());
+ ensureSuccess("PUT " + target.uri(), response.statusCode(), response.body());
+ }
+
+ /**
+ * Returns the object body; the caller is responsible for closing it.
+ */
+ public InputStream getObject(String bucket, String key) throws IOException, InterruptedException {
+ var target = target(bucket, key);
+ var request = signed("GET", target, AwsV4Signer.EMPTY_PAYLOAD_SHA256)
+ .GET().build();
+ var response = http.send(request, HttpResponse.BodyHandlers.ofInputStream());
+ if (response.statusCode() / 100 != 2) {
+ String body;
+ try (var in = response.body()) {
+ body = new String(in.readNBytes(4096), StandardCharsets.UTF_8);
+ }
+ throw new S3Exception("GET " + target.uri(), response.statusCode(), body);
+ }
+ return response.body();
+ }
+
+ public void deleteObject(String bucket, String key) throws IOException, InterruptedException {
+ var target = target(bucket, key);
+ var request = signed("DELETE", target, AwsV4Signer.EMPTY_PAYLOAD_SHA256)
+ .DELETE().build();
+ var response = http.send(request, HttpResponse.BodyHandlers.ofString());
+ ensureSuccess("DELETE " + target.uri(), response.statusCode(), response.body());
+ }
+
+ @Override
+ public void close() {
+ http.close();
+ }
+
+ private record Target(URI uri, String host, String encodedPath) {
+ }
+
+ private Target target(String bucket, String key) {
+ var host = pathStyle ? hostPort : bucket + "." + hostPort;
+ var path = new StringBuilder();
+ if (pathStyle) {
+ path.append('/').append(URLEncoder.encode(bucket, Charset.defaultCharset()));
+ }
+ if (key != null) {
+ for (var segment : key.split("/", -1)) {
+ path.append('/').append(URLEncoder.encode(segment, Charset.defaultCharset()));
+ }
+ }
+ if (path.isEmpty()) {
+ path.append('/');
+ }
+ return new Target(URI.create(scheme + "://" + host + path), host, path.toString());
+ }
+
+ /**
+ * Host is signed but never set explicitly — HttpClient forbids that and
+ * derives it from the URI; {@link #hostPort} mirrors that derivation.
+ */
+ private HttpRequest.Builder signed(String method, Target target, String payloadHash) {
+ var amzDate = AMZ_DATE.format(Instant.now());
+ var headers = new TreeMap();
+ headers.put("host", target.host());
+ headers.put("x-amz-content-sha256", payloadHash);
+ headers.put("x-amz-date", amzDate);
+ var authorization = AwsV4Signer.authorization(method, target.encodedPath(), "",
+ headers, payloadHash, amzDate, region, "s3", accessKey, secretKey);
+ return HttpRequest.newBuilder(target.uri())
+ .header("x-amz-content-sha256", payloadHash)
+ .header("x-amz-date", amzDate)
+ .header("authorization", authorization);
+ }
+
+ private static void ensureSuccess(String request, int statusCode, String body) throws S3Exception {
+ if (statusCode / 100 != 2) {
+ throw new S3Exception(request, statusCode, body);
+ }
+ }
+
+ private static String sha256Hex(Path file) throws IOException {
+ var digest = AwsV4Signer.sha256();
+ try (var in = Files.newInputStream(file)) {
+ var buffer = new byte[64 * 1024];
+ int read;
+ while ((read = in.read(buffer)) != -1) {
+ digest.update(buffer, 0, read);
+ }
+ }
+ return HexFormat.of().formatHex(digest.digest());
+ }
+}
diff --git a/src/main/java/io/ib67/sfcraft/strategy/S3BackupStrategy.java b/src/main/java/io/ib67/sfcraft/strategy/S3BackupStrategy.java
index ff625c9..d84c317 100644
--- a/src/main/java/io/ib67/sfcraft/strategy/S3BackupStrategy.java
+++ b/src/main/java/io/ib67/sfcraft/strategy/S3BackupStrategy.java
@@ -3,15 +3,11 @@ package io.ib67.sfcraft.strategy;
import io.ib67.sfcraft.Backup;
import io.ib67.sfcraft.bundler.BundleReader;
import io.ib67.sfcraft.config.StorageOption;
+import io.ib67.sfcraft.s3.SimpleS3Client;
import lombok.SneakyThrows;
import lombok.extern.log4j.Log4j2;
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import software.amazon.awssdk.core.exception.SdkException;
-import software.amazon.awssdk.regions.Region;
-import software.amazon.awssdk.services.s3.S3Client;
-import java.net.URI;
+import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
@@ -20,22 +16,17 @@ import java.util.Objects;
@Log4j2
public class S3BackupStrategy implements BackupStrategy {
protected final StorageOption.S3 option;
- protected final S3Client s3;
+ protected final SimpleS3Client s3;
protected final Path temporaryDownloadPath;
@SneakyThrows
public S3BackupStrategy(StorageOption.S3 option, Path tempDownloadPath) {
this.option = option;
- var credential = AwsBasicCredentials.create(option.accessKey(), option.secretKey());
- this.s3 = S3Client.builder()
- .region(Region.of(option.region()))
- .forcePathStyle(option.enforcePathStyle())
- .endpointOverride(URI.create(option.endpoint()))
- .credentialsProvider(StaticCredentialsProvider.create(credential))
- .build();
+ this.s3 = new SimpleS3Client(option.endpoint(), option.region(),
+ option.accessKey(), option.secretKey(), option.enforcePathStyle());
log.info("Testing S3 credentials");
// throws exception if the bucket neither inaccessible nor not exist
- s3.headBucket(b -> b.bucket(option.bucket()));
+ s3.headBucket(option.bucket());
this.temporaryDownloadPath = Objects.requireNonNull(tempDownloadPath);
if (Files.notExists(temporaryDownloadPath))
Files.createDirectories(temporaryDownloadPath);
@@ -49,15 +40,13 @@ public class S3BackupStrategy implements BackupStrategy {
int attempt = 0;
while (true) {
try {
- s3.putObject(o -> o.bucket(option.bucket())
- .contentType("application/octet-stream")
- .key(object), bundle);
+ s3.putObject(option.bucket(), object, bundle, "application/octet-stream");
return new Backup(bundle.getFileName().toString(),
object, "s3",
source.toString(),
subject, incremental, createdAt,
Files.size(bundle));
- } catch (SdkException e) {
+ } catch (IOException e) {
if (attempt >= option.retryAmount()) {
throw new IllegalStateException("Cannot upload " + bundle + " to S3 after "
+ attempt + " retries", e);
@@ -75,7 +64,7 @@ public class S3BackupStrategy implements BackupStrategy {
public void recoverBackup(Backup backup, Path restorePath, boolean ignoreDeletions) {
var target = temporaryDownloadPath.resolve("s3_" + System.currentTimeMillis());
try {
- try (var resp = s3.getObject(b -> b.bucket(option.bucket()).key(backup.backupKey()));
+ try (var resp = s3.getObject(option.bucket(), backup.backupKey());
var fs = Files.newOutputStream(target, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
resp.transferTo(fs);
}
@@ -88,7 +77,7 @@ public class S3BackupStrategy implements BackupStrategy {
@Override
@SneakyThrows
public void deleteBackup(Backup backup) {
- s3.deleteObject(b -> b.bucket(option.bucket()).key(backup.backupKey()).build());
+ s3.deleteObject(option.bucket(), backup.backupKey());
}
@Override
diff --git a/src/test/java/io/ib67/sfcraft/s3/AwsV4SignerTest.java b/src/test/java/io/ib67/sfcraft/s3/AwsV4SignerTest.java
new file mode 100644
index 0000000..312d35e
--- /dev/null
+++ b/src/test/java/io/ib67/sfcraft/s3/AwsV4SignerTest.java
@@ -0,0 +1,59 @@
+package io.ib67.sfcraft.s3;
+
+import org.junit.jupiter.api.Test;
+
+import java.net.URLEncoder;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.TreeMap;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Test vectors from
+ * https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+ */
+class AwsV4SignerTest {
+ private static final String ACCESS_KEY = "AKIAIOSFODNN7EXAMPLE";
+ private static final String SECRET_KEY = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
+
+ @Test
+ void getObjectExample() {
+ var headers = new TreeMap();
+ headers.put("host", "examplebucket.s3.amazonaws.com");
+ headers.put("range", "bytes=0-9");
+ headers.put("x-amz-content-sha256", AwsV4Signer.EMPTY_PAYLOAD_SHA256);
+ headers.put("x-amz-date", "20130524T000000Z");
+
+ var authorization = AwsV4Signer.authorization("GET", "/test.txt", "",
+ headers, AwsV4Signer.EMPTY_PAYLOAD_SHA256,
+ "20130524T000000Z", "us-east-1", "s3", ACCESS_KEY, SECRET_KEY);
+
+ assertEquals("AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,"
+ + "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date,"
+ + "Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41",
+ authorization);
+ }
+
+ @Test
+ void putObjectExample() {
+ var payloadHash = AwsV4Signer.sha256Hex("Welcome to Amazon S3.".getBytes(StandardCharsets.UTF_8));
+ assertEquals("44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072", payloadHash);
+
+ var headers = new TreeMap();
+ headers.put("date", "Fri, 24 May 2013 00:00:00 GMT");
+ headers.put("host", "examplebucket.s3.amazonaws.com");
+ headers.put("x-amz-content-sha256", payloadHash);
+ headers.put("x-amz-date", "20130524T000000Z");
+ headers.put("x-amz-storage-class", "REDUCED_REDUNDANCY");
+
+ var authorization = AwsV4Signer.authorization("PUT", "/test%24file.text", "",
+ headers, payloadHash,
+ "20130524T000000Z", "us-east-1", "s3", ACCESS_KEY, SECRET_KEY);
+
+ assertEquals("AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,"
+ + "SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,"
+ + "Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd",
+ authorization);
+ }
+}