cloud/redapricot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

impl connection recovery

Browse Source
This commit is contained in:
iceBear67
2026-08-15 17:31:35 +08:00
parent e63a34d53a
commit 7bd84af48d
33 changed files with 3858 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/config.example.json
+6 -1
View File
@@ -4,5 +4,10 @@
"timestampWindowMs": 30000,
"pendingTimeoutMs": 10000,
"streamWindowBytes": 262144,
"sessionIdleTimeoutMs": 90000
"sessionIdleTimeoutMs": 90000,
"streamResume": true,
"resumeGraceMs": 20000,
"maxParkedStreams": 256,
"statsIntervalMs": 0,
"registrationGraceMs": 15000
}
server/src/main/java/io/icybear/redapricot/Config.java
+31 -2
View File
@@ -13,7 +13,13 @@ public record Config(
long timestampWindowMs,
long pendingTimeoutMs,
int streamWindowBytes,
long sessionIdleTimeoutMs
long sessionIdleTimeoutMs,
boolean streamResume,
long resumeGraceMs,
int maxParkedStreams,
long maxParkedBytes,
long statsIntervalMs,
long registrationGraceMs
) {
public static Config load(Path file) throws Exception {
JsonObject json = new JsonObject(Files.readString(file));
@@ -30,6 +36,15 @@ public record Config(
int window = json.getInteger("streamWindowBytes", Protocol.DEFAULT_STREAM_WINDOW);
window = Math.max(Protocol.MIN_STREAM_WINDOW, Math.min(Protocol.MAX_STREAM_WINDOW, window));
boolean resume = json.getBoolean("streamResume", Boolean.TRUE);
// A parked stream can hold up to one window of unacked bytes plus one of
// parked player bytes, for the whole grace period, and nothing else
// bounds how many streams park at once — so anyone able to kill worker
// conns is otherwise a cheap memory amplifier. The default admits ~256
// hanging players at the default window.
int maxParked = json.getInteger("maxParkedStreams", 256);
long maxParkedBytes = json.getLong("maxParkedBytes", (long) maxParked * 2 * window);
return new Config(
host,
port,
@@ -39,6 +54,20 @@ public record Config(
window,
// Comfortably above the client's default 20s ping interval;
// 0 disables the watchdog.
json.getLong("sessionIdleTimeoutMs", 90_000L));
json.getLong("sessionIdleTimeoutMs", 90_000L),
resume,
// Must exceed the client's own grace by at least one dial, or the
// hub drops a player while its client is still mid-reattach. The
// value is advertised in SessionReady precisely so the client can
// clamp itself under it rather than rely on matching config.
json.getLong("resumeGraceMs", 20_000L),
maxParked,
maxParkedBytes,
json.getLong("statsIntervalMs", 0L),
// Long enough to cover a client's control-session reconnect
// (its backoff caps at 10s) without holding a player so long
// that they give up anyway; 0 disables and restores the old
// behaviour of dropping routes the moment a session closes.
json.getLong("registrationGraceMs", 15_000L));
}
}
server/src/main/java/io/icybear/redapricot/Hub.java
+333 -22
View File
@@ -7,10 +7,16 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.nio.charset.StandardCharsets;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadLocalRandom;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
@@ -21,6 +27,7 @@ import java.util.regex.PatternSyntaxException;
*/
public final class Hub {
private static final Logger LOG = LogManager.getLogger("redapricot.hub");
private static final SecureRandom RNG = new SecureRandom();
public final Vertx vertx;
public final Config config;
@@ -30,11 +37,40 @@ public final class Hub {
private final Map<String, Registration> patterns = new ConcurrentHashMap<>();
private final Map<String, PendingPlayer> pending = new ConcurrentHashMap<>();
/** A compiled routing pattern and the control session that registered it. */
private record Registration(Pattern regex, ControlSession session) {}
/**
* Every tunneled player, keyed by its current CID, whether live or parked.
* Keeping parked streams here rather than on the worker conn is the whole
* point: a stream's identity is the player, and state that dies with the
* connection cannot survive that connection dying. Insertion-ordered so the
* parked cap can evict the oldest first.
*/
private final Map<String, PlayerStream> streams = new LinkedHashMap<>();
private int parkedCount;
private long parkedBytes;
/** A successful match: the registered pattern that matched and its owning session. */
public record Match(String pattern, ControlSession session) {}
/**
* A compiled routing pattern and the control session that registered it.
*
* <p>{@code session} is null while the registration is <b>orphaned</b> — its
* client's control session has closed but the route is held open until
* {@code orphanDeadline} in case the client reconnects. Players matching an
* orphaned route are hung rather than refused.
*/
private record Registration(Pattern regex, ControlSession session, long orphanDeadline) {
Registration(Pattern regex, ControlSession session) {
this(regex, session, 0);
}
boolean orphaned() {
return session == null;
}
}
/**
* A successful match: the registered pattern that matched and its owning
* session, which is null when the route is orphaned (§ control-outage hang).
*/
public record Match(String pattern, ControlSession session, long orphanDeadline) {}
public Hub(Vertx vertx, Config config) {
this.vertx = vertx;
@@ -64,9 +100,29 @@ public final class Hub {
}
patterns.put(pattern, new Registration(regex, session));
LOG.info("registered pattern '{}' -> {}", pattern, session.id());
replayAwaiting(pattern, session);
return Protocol.REGISTER_OK;
}
/**
* Deliver the control requests held while this pattern had no live session.
*
* <p>These players connected during the client's reconnect and were hung
* instead of refused; the request was never sent, so it is sent now. Each
* moves from "waiting for a route" to the ordinary "waiting for a worker",
* which means swapping its deadline over to {@code pendingTimeoutMs}.
*/
private void replayAwaiting(String pattern, ControlSession session) {
for (PendingPlayer p : pending.values()) {
if (!p.isAwaitingSession() || !p.getPattern().equals(pattern)) continue;
p.setAwaitingSession(false);
p.setOwner(session);
rearm(p, config.pendingTimeoutMs());
session.sendControlRequest(p.getCid(), p.getPattern(), p.getPlayerIp(), p.getPlayerPort());
LOG.info("replayed control request for hung player {} on session {}", p.getCidHex(), session.id());
}
}
public void unregister(String pattern, ControlSession session) {
// Remove only if this session still owns the pattern (a newer session may have taken it).
patterns.computeIfPresent(pattern, (k, reg) -> reg.session() == session ? null : reg);
@@ -80,36 +136,140 @@ public final class Hub {
public Match match(String address) {
String host = normalizeAddress(address);
for (Map.Entry<String, Registration> e : patterns.entrySet()) {
if (e.getValue().regex().matcher(host).matches()) {
return new Match(e.getKey(), e.getValue().session());
Registration reg = e.getValue();
if (reg.regex().matcher(host).matches()) {
return new Match(e.getKey(), reg.session(), reg.orphanDeadline());
}
}
return null;
}
/** Drop every pattern owned by a (closing) session, plus any players still pending for it. */
/**
* A control session closed. Its routes are kept as <b>orphaned</b> for
* {@code registrationGraceMs}, and the players waiting on them are hung
* rather than dropped.
*
* <p>Without this, a client's reconnect — half a second at best, ten at worst
* once its backoff has grown — is a window in which every arriving player is
* told there is no such server, even though the tunnel is seconds from being
* back. The players already tunneled are unaffected either way; they ride
* worker conns, which a control-session close never touches.
*
* <p>A grace of 0 restores the old behaviour exactly.
*/
public void removeSession(ControlSession session) {
patterns.entrySet().removeIf(e -> e.getValue().session() == session);
long grace = config.registrationGraceMs();
if (grace <= 0) {
patterns.entrySet().removeIf(e -> e.getValue().session() == session);
pending.values().removeIf(p -> {
if (p.getOwner() != session) return false;
cancelTimer(p);
p.getSocket().close();
LOG.info("dropping pending player {} (control session {} closed)", p.getCidHex(), session.id());
return true;
});
return;
}
long deadline = System.currentTimeMillis() + grace;
int orphaned = 0;
for (Map.Entry<String, Registration> e : patterns.entrySet()) {
Registration reg = e.getValue();
if (reg.session() != session) continue;
e.setValue(new Registration(reg.regex(), null, deadline));
orphaned++;
}
// A player that was already matched is in the same position: its request
// went to a session that will never answer, so it waits for the route to
// come back and is then replayed like any other.
int hung = 0;
for (PendingPlayer p : pending.values()) {
if (p.getOwner() != session) continue;
p.setOwner(null);
p.setAwaitingSession(true);
rearm(p, grace);
hung++;
}
if (orphaned > 0 || hung > 0) {
LOG.info("control session {} closed; holding {} route(s) and {} player(s) for {}ms",
session.id(), orphaned, hung, grace);
vertx.setTimer(grace, id -> expireOrphans());
}
}
/** Drop routes whose grace ran out, and the players still hung on them. */
private void expireOrphans() {
long now = System.currentTimeMillis();
Set<String> gone = new HashSet<>();
patterns.entrySet().removeIf(e -> {
Registration reg = e.getValue();
if (!reg.orphaned() || reg.orphanDeadline() > now) return false;
gone.add(e.getKey());
return true;
});
if (gone.isEmpty()) return;
LOG.info("dropping {} orphaned route(s) not reclaimed within the grace period", gone.size());
pending.values().removeIf(p -> {
if (p.getOwner() != session) return false;
if (p.getTimerId() >= 0) vertx.cancelTimer(p.getTimerId());
if (!p.isAwaitingSession() || !gone.contains(p.getPattern())) return false;
cancelTimer(p);
p.getSocket().close();
LOG.info("dropping pending player {} (control session {} closed)", p.getCidHex(), session.id());
LOG.info("dropping hung player {} (route '{}' never came back)", p.getCidHex(), p.getPattern());
return true;
});
}
// ---- pending players ----
/**
* Mint a takeover capability. The CID is the only thing authorizing a client
* to claim a player (and, with resumption, to reclaim one), so it comes from
* a cryptographic source rather than ThreadLocalRandom — a predictable value
* would be a session-hijacking primitive.
*/
public byte[] newCid() {
byte[] cid = new byte[Protocol.CID_LEN];
ThreadLocalRandom.current().nextBytes(cid);
RNG.nextBytes(cid);
return cid;
}
public void addPending(PendingPlayer p) {
pending.put(p.getCidHex(), p);
p.setTimerId(vertx.setTimer(config.pendingTimeoutMs(), id -> {
rearm(p, config.pendingTimeoutMs());
}
/**
* Hang a player whose route is orphaned: hold it until a client re-registers
* the pattern, at which point {@link #replayAwaiting} delivers the control
* request that was never sent.
*
* @param deadline wall-clock millis at which the route's grace runs out
*/
public void addAwaiting(PendingPlayer p, long deadline) {
p.setOwner(null);
p.setAwaitingSession(true);
pending.put(p.getCidHex(), p);
rearm(p, Math.max(1, deadline - System.currentTimeMillis()));
LOG.info("holding player {} for '{}': route is orphaned, waiting for its client",
p.getCidHex(), p.getPattern());
}
public PendingPlayer takePending(byte[] cid) {
String hex = Hex.encode(cid);
PendingPlayer p = pending.remove(hex);
if (p != null) cancelTimer(p);
return p;
}
public void removePending(String cidHex) {
PendingPlayer p = pending.remove(cidHex);
if (p != null) cancelTimer(p);
}
/** Replace a pending player's deadline, cancelling whatever it had. */
private void rearm(PendingPlayer p, long delayMs) {
cancelTimer(p);
p.setTimerId(vertx.setTimer(delayMs, id -> {
PendingPlayer removed = pending.remove(p.getCidHex());
if (removed != null) {
LOG.warn("pending player {} timed out", p.getCidHex());
@@ -118,16 +278,167 @@ public final class Hub {
}));
}
public PendingPlayer takePending(byte[] cid) {
String hex = Hex.encode(cid);
PendingPlayer p = pending.remove(hex);
if (p != null && p.getTimerId() >= 0) vertx.cancelTimer(p.getTimerId());
return p;
private void cancelTimer(PendingPlayer p) {
if (p.getTimerId() >= 0) {
vertx.cancelTimer(p.getTimerId());
p.setTimerId(-1);
}
}
public void removePending(String cidHex) {
PendingPlayer p = pending.remove(cidHex);
if (p != null && p.getTimerId() >= 0) vertx.cancelTimer(p.getTimerId());
// ---- tunneled streams & resumption (§7.5) ----
/** Register a stream that has just been bound to a worker conn. */
public void addStream(PlayerStream st) {
streams.put(st.cidHex, st);
}
/** Forget a stream for good; its player socket is gone or going. */
public void removeStream(PlayerStream st) {
if (streams.remove(st.cidHex) == null) return;
if (st.parked) unpark(st);
st.unacked.clear();
}
/** Look up a stream by the CID a client presented, live or parked. */
public PlayerStream streamByCid(byte[] cid) {
return streams.get(Hex.encode(cid));
}
/**
* Player bytes arrived: hand them to whichever conn currently carries the
* stream. Routing here rather than from the conn that installed the socket
* handler is what lets a stream change conns without rebinding handlers.
*/
public void onPlayerData(PlayerStream st, io.vertx.core.buffer.Buffer buf) {
WorkerConn w = st.worker;
if (w != null) {
w.playerData(st, buf);
return;
}
// Parked, so there is nowhere to send: hold the bytes in order and make
// sure the socket really is stopped. The player is paused on park, but a
// batch already in flight can still land here.
if (st.pendingUp == null) st.pendingUp = io.vertx.core.buffer.Buffer.buffer();
st.pendingUp.appendBuffer(buf);
st.player.pause();
}
/** The player hung up. Drop the stream everywhere and tell the client if it is still bound. */
public void onPlayerGone(PlayerStream st) {
WorkerConn w = st.worker;
removeStream(st);
if (w != null) w.playerGone(st);
}
/**
* Hang a player whose worker conn died, instead of closing it.
*
* <p>Only the tunnel leg failed — the player socket is still perfectly good —
* so it is paused and held until the client reattaches the stream over a
* fresh conn. The deadline is absolute and fixed at the first park: re-arming
* it on each park would let a flapping client hold a player forever.
*
* @return false if the stream cannot be parked and must be closed instead
*/
public boolean park(PlayerStream st) {
if (!st.resumable || st.parked) return false;
long now = System.currentTimeMillis();
if (st.graceDeadline == 0) st.graceDeadline = now + config.resumeGraceMs();
long remaining = st.graceDeadline - now;
if (remaining <= 0) return false;
st.parked = true;
st.worker = null;
// Explicitly, not as a side effect of the window filling: an idle stream
// has a wide-open window, so nothing else would stop the next keepalive
// walking into a send on a dead connection.
st.player.pause();
st.pausedForAggregate = false; // that conn's drain handler will never fire again
parkedCount++;
parkedBytes += st.parkedBytes();
st.timerId = vertx.setTimer(remaining, id -> {
LOG.info("parked player {} not reclaimed within grace; closing", st.cidHex);
removeStream(st);
st.player.close();
});
enforceParkedCaps(st);
return true;
}
/**
* Reclaim a parked stream. Returns null when no stream is parked under this
* CID; the caller distinguishes "never heard of it" from "still bound
* elsewhere" via {@link #streamByCid}.
*/
public PlayerStream takeParked(byte[] cid) {
PlayerStream st = streams.get(Hex.encode(cid));
if (st == null || !st.parked) return null;
unpark(st);
return st;
}
/** Re-key a stream to the freshly minted CID handed out in RESUME_ACK. */
public void rekeyStream(PlayerStream st, byte[] cid) {
streams.remove(st.cidHex);
st.cid = cid;
st.cidHex = Hex.encode(cid);
streams.put(st.cidHex, st);
}
private void unpark(PlayerStream st) {
if (!st.parked) return;
st.parked = false;
parkedCount--;
parkedBytes -= st.parkedBytes();
if (parkedBytes < 0) parkedBytes = 0;
if (st.timerId >= 0) {
vertx.cancelTimer(st.timerId);
st.timerId = -1;
}
}
/**
* Bound what hanging players may cost. Each holds up to a window of unsent
* bytes plus a window of parked ones for the whole grace period, and nothing
* else limits how many park at once — so anyone able to kill worker conns is
* otherwise a cheap memory amplifier. Oldest first, since they have the least
* grace left to be reclaimed in.
*/
private void enforceParkedCaps(PlayerStream keep) {
if (parkedCount <= config.maxParkedStreams() && parkedBytes <= config.maxParkedBytes()) return;
List<PlayerStream> evict = new ArrayList<>();
Iterator<PlayerStream> it = streams.values().iterator();
while (it.hasNext() && (parkedCount - evict.size() > config.maxParkedStreams()
|| parkedBytes > config.maxParkedBytes())) {
PlayerStream st = it.next();
if (!st.parked || st == keep) continue;
evict.add(st);
parkedBytes -= st.parkedBytes();
}
for (PlayerStream st : evict) {
LOG.warn("parked-stream cap reached; dropping hanging player {}", st.cidHex);
parkedBytes += st.parkedBytes(); // removeStream subtracts it again
removeStream(st);
st.player.close();
}
}
/** Live and parked stream counts, for the periodic stats line. */
public int streamCount() {
return streams.size();
}
public int parkedCount() {
return parkedCount;
}
public long parkedBytes() {
return parkedBytes;
}
public int patternCount() {
return patterns.size();
}
// ---- helpers ----
server/src/main/java/io/icybear/redapricot/HubConnection.java
+33 -12
View File
@@ -168,6 +168,10 @@ public final class HubConnection {
// Heartbeat is optional: accept it only when the client offered it, so
// older clients keep working (they just lose silent-path detection).
boolean heartbeat = (flags & Protocol.FLAG_WORKER_HEARTBEAT) != 0;
// Resumption likewise. Parking a stream for a client that will never
// reattach is strictly worse than closing it — the player hangs for the
// whole grace instead of failing fast — so this bit gates the park path.
boolean resume = hub.config.streamResume() && (flags & Protocol.FLAG_STREAM_RESUME) != 0;
// REKEY = Rand || Timestamp(I64 big-endian). Magic is excluded.
byte[] rekey = new byte[randLen + 8];
@@ -181,13 +185,20 @@ public final class HubConnection {
frames.switchCiphers(
Crypto.decryptCipher(rekey, Crypto.DIR_C2S),
Crypto.encryptCipher(rekey, Crypto.DIR_S2C));
// Echo the accepted flags plus our own receive window.
int accepted = Protocol.FLAG_STREAM_FC | (heartbeat ? Protocol.FLAG_WORKER_HEARTBEAT : 0);
frames.send(new ProtoWriter()
// Echo the accepted flags plus our own receive window. When resumption is
// accepted, our grace period follows: the client clamps its own retry
// budget under it, which turns a cross-config invariant ("the hub must
// wait longer than the client retries") into a negotiated one that
// operator skew cannot break.
int accepted = Protocol.FLAG_STREAM_FC
| (heartbeat ? Protocol.FLAG_WORKER_HEARTBEAT : 0)
| (resume ? Protocol.FLAG_STREAM_RESUME : 0);
ProtoWriter ready = new ProtoWriter()
.u8(Protocol.CTL_SESSION_READY)
.varInt(accepted)
.varInt(hub.config.streamWindowBytes())
.toBytes());
.varInt(hub.config.streamWindowBytes());
if (resume) ready.varInt((int) hub.config.resumeGraceMs());
frames.send(ready.toBytes());
if (magic == Protocol.MAGIC_CONTROL) {
ControlSession session = new ControlSession(hub, frames, id);
@@ -195,10 +206,11 @@ public final class HubConnection {
closeCleanup = session::onClose;
LOG.info("{} control session established (heartbeat {})", id, heartbeat);
} else if (magic == Protocol.MAGIC_WORKER) {
WorkerConn worker = new WorkerConn(hub, frames, id, peerWindow, hub.config.streamWindowBytes());
WorkerConn worker = new WorkerConn(hub, frames, id, peerWindow, hub.config.streamWindowBytes(), resume);
frames.setHandler(worker::onFrame);
closeCleanup = worker::onClose;
LOG.info("{} worker conn established (peer window {}, heartbeat {})", id, peerWindow, heartbeat);
LOG.info("{} worker conn established (peer window {}, heartbeat {}, resume {})",
id, peerWindow, heartbeat, resume);
} else {
LOG.warn("{} bad magic {}; closing", id, magic);
frames.close();
@@ -259,12 +271,21 @@ public final class HubConnection {
socket.pause();
Buffer buffered = hs.copy(); // handshake + any pipelined bytes, forwarded verbatim
PendingPlayer p = new PendingPlayer(cid, cidHex, socket, buffered, pattern, ip, port, session);
hub.addPending(p);
PendingPlayer p = new PendingPlayer(cid, cidHex, socket, buffered, pattern, ip, port);
closeCleanup = () -> hub.removePending(cidHex);
session.sendControlRequest(cid, pattern, ip, port);
LOG.info("{} player {}:{} host '{}' matched pattern '{}' cid={}",
id, ip, port, host, pattern, cidHex);
if (session == null) {
// The route is orphaned: its client's control session has closed and
// has not come back yet. Hold the player rather than telling it there
// is no such server — the request is replayed the moment a client
// re-registers the pattern.
hub.addAwaiting(p, matched.orphanDeadline());
} else {
p.setOwner(session);
hub.addPending(p);
session.sendControlRequest(cid, pattern, ip, port);
}
LOG.info("{} player {}:{} host '{}' matched pattern '{}' cid={}{}",
id, ip, port, host, pattern, cidHex, session == null ? " (held: route orphaned)" : "");
}
}
server/src/main/java/io/icybear/redapricot/HubServer.java
+17
View File
@@ -35,10 +35,27 @@ public final class HubServer extends AbstractVerticle {
if (ar.succeeded()) {
LOG.info("redapricot hub listening on {}:{}", config.host(), ar.result().actualPort());
LOG.info("PSK handshake address: {}", hub.pskAddress);
armStats();
startPromise.complete();
} else {
startPromise.fail(ar.cause());
}
});
}
/**
* Periodic one-line snapshot of what the hub is holding. Off unless
* statsIntervalMs is set, so it costs nothing by default.
*
* <p>Parked streams and the bytes they retain are the numbers worth watching:
* they are the memory stream resumption trades for keeping players connected,
* and the first place a resumption problem shows up as a trend.
*/
private void armStats() {
long interval = config.statsIntervalMs();
if (interval <= 0) return;
vertx.setPeriodic(interval, id -> LOG.info(
"stats streams={} parked={} parkedBytes={} patterns={}",
hub.streamCount(), hub.parkedCount(), hub.parkedBytes(), hub.patternCount()));
}
}
server/src/main/java/io/icybear/redapricot/PendingPlayer.java
+16 -1
View File
@@ -17,7 +17,22 @@ public final class PendingPlayer {
private final String pattern;
private final String playerIp;
private final int playerPort;
private final ControlSession owner; // control session this player was routed to
/**
* Control session this player was routed to, or null while the route is
* orphaned. Mutable because a hung player is rebound to whichever session
* re-registers its pattern.
*/
@Setter
private ControlSession owner;
/**
* Hung waiting for a control session to come back, rather than waiting for a
* worker to claim it. No ControlRequest has been delivered yet, so this
* player is the hub's to replay once a route reappears.
*/
@Setter
private boolean awaitingSession;
@Setter
private long timerId = -1;
server/src/main/java/io/icybear/redapricot/PlayerStream.java
+88
View File
@@ -0,0 +1,88 @@
package io.icybear.redapricot;
import io.vertx.core.buffer.Buffer;
import io.vertx.core.net.NetSocket;
/**
* One tunneled player: the player socket plus the flow-control state of the mux
* stream carrying it (PROTOCOL.md §7.3).
*
* <p>This is deliberately <em>not</em> owned by {@link WorkerConn}. A stream's
* identity is the player, not the connection it happens to ride: the worker conn
* is a replaceable transport, and state that dies with it cannot be recovered
* when it drops.
*
* <p>Confined to the hub's single event loop, so the mutable fields need no
* synchronization.
*/
public final class PlayerStream {
/** Capability that authorized the takeover; also the resume key. Re-minted on each reattach. */
byte[] cid;
String cidHex;
final NetSocket player;
/** Registered pattern that matched, echoed to the client. */
final String pattern;
final String playerIp;
final int playerPort;
/** The conn currently carrying this stream, and its id there. */
WorkerConn worker;
int sid;
/** Budget for player -> client DATA. */
int sendWnd;
/** Player bytes awaiting send window; the player is paused while non-null. */
Buffer pendingUp;
/** client -> player bytes flushed but not yet granted back. */
int credited;
// Pause reasons. Vert.x pause() is a flag rather than a counter, so a socket
// can be paused for several reasons at once and must only be resumed once
// none of them hold — see WorkerConn#maybeResumePlayer.
boolean pausedForWindow; // this stream's send window is exhausted
boolean pausedForAggregate; // the shared worker socket's write queue is full
boolean parked; // the worker conn died; hanging until a reattach
/** Whether the conn carrying this stream negotiated resumption (§7.5). */
boolean resumable;
// Resumption bookkeeping (§7.5). Three distinct offsets, and conflating them
// is the classic mistake: what to retransmit is measured from what the peer
// *accepted*, while the flow-control window is measured from what it
// *credited*. The gap between the two is credit still owed.
long sentOffset; // bytes handed to the wire
long ackedOffset; // running sum of WND deltas received
long acceptedOffset; // client -> player bytes taken off the wire
long deliveredOffset; // client -> player bytes actually written to the socket
final UnackedBytes unacked = new UnackedBytes();
/**
* Absolute wall-clock deadline for reattaching, fixed at the first park. Not
* re-armed on a later park: a flapping hub would otherwise keep extending it
* and hang the player indefinitely.
*/
long graceDeadline;
long timerId = -1;
PlayerStream(PendingPlayer p, WorkerConn worker, int sid, int sendWnd) {
this.cid = p.getCid();
this.cidHex = p.getCidHex();
this.player = p.getSocket();
this.pattern = p.getPattern();
this.playerIp = p.getPlayerIp();
this.playerPort = p.getPlayerPort();
this.worker = worker;
this.sid = sid;
this.sendWnd = sendWnd;
}
/** Whether the player socket should be flowing right now. */
boolean shouldFlow() {
return !pausedForWindow && !pausedForAggregate && !parked;
}
/** Roughly how much this stream holds while parked, for the hub-wide cap. */
int parkedBytes() {
return unacked.length() + (pendingUp != null ? pendingUp.length() : 0);
}
}
server/src/main/java/io/icybear/redapricot/Protocol.java
+22
View File
@@ -34,15 +34,37 @@ public final class Protocol {
public static final int MUX_WND = 0x04; // per-stream flow-control credit grant
public static final int MUX_PING = 0x05; // liveness probe, StreamID 0
public static final int MUX_PONG = 0x06; // liveness reply, echoes the nonce
/** Reattach a parked stream to this conn: CID + the client's accepted offset (§7.5). */
public static final int MUX_RESUME = 0x07;
/** Hub's answer to RESUME: its accepted offset plus a freshly minted CID. */
public static final int MUX_RESUME_ACK = 0x08;
/** Reserved stream id for connection-scoped mux frames (PING/PONG). Streams start at 1. */
public static final int MUX_CTL_SID = 0;
// RST reason codes (optional trailing byte; absence means "unspecified").
// Distinguishing them matters for resume: "unknown stream" is terminal,
// "already bound" means a racing attempt won and this one must not tear down.
public static final int RST_UNSPECIFIED = 0x00;
public static final int RST_UNKNOWN_STREAM = 0x01; // CID unknown, expired, or hub restarted
public static final int RST_ALREADY_BOUND = 0x02; // another RESUME won the race
public static final int RST_RESUME_ABANDONED = 0x03;
public static final int RST_FLOW_CONTROL = 0x04;
public static final int RST_DIAL_FAILED = 0x05;
// Session-establishment feature flags (trailing VarInt on the Rekey message,
// echoed after the SessionReady type byte when accepted).
public static final int FLAG_STREAM_FC = 0x01;
/** Mux-level PING/PONG on worker conns, so a silently dropped path is detected. */
public static final int FLAG_WORKER_HEARTBEAT = 0x02;
/**
* Stream resumption (§7.5): on a worker-conn drop the hub hangs the player
* socket instead of closing it, and the client reattaches the stream
* byte-exactly over a fresh conn. When accepted, the hub appends its resume
* grace period to SessionReady so the client can bound its own retry budget
* against it.
*/
public static final int FLAG_STREAM_RESUME = 0x04;
// Per-stream flow-control window bounds (bytes).
public static final int DEFAULT_STREAM_WINDOW = 256 * 1024;
server/src/main/java/io/icybear/redapricot/UnackedBytes.java
+95
View File
@@ -0,0 +1,95 @@
package io.icybear.redapricot;
import io.vertx.core.buffer.Buffer;
import java.util.ArrayDeque;
import java.util.Deque;
/**
* The bytes a stream has sent but the client has not yet credited — exactly the
* region a reattach may have to retransmit (PROTOCOL.md §7.5).
*
* <p>It needs no cap of its own: credit is only granted as bytes reach the
* client's destination socket, so flow control already bounds the outstanding
* region to one window. That is what makes byte-exact resumption affordable.
*
* <p>A deque of the chunks already materialized by the send path, rather than one
* growing {@link Buffer}: appending to a Buffer reallocates and recopies as it
* grows, which would add a second per-byte copy to the whole upstream path. Here
* retention is free — the chunk was allocated to be sent anyway.
*/
final class UnackedBytes {
private final Deque<byte[]> chunks = new ArrayDeque<>();
private int head; // bytes of the first chunk already credited
private long base; // stream offset of the first live byte
private int length; // live bytes across all chunks
int length() {
return length;
}
long base() {
return base;
}
/** Offset one past the last byte handed to the wire. */
long end() {
return base + length;
}
void append(byte[] chunk) {
if (chunk.length == 0) return;
chunks.addLast(chunk);
length += chunk.length;
}
/** Drop everything the client has credited up to {@code off}. */
void advance(long off) {
long drop = off - base;
if (drop <= 0) return;
if (drop > length) drop = length; // only from a peer crediting bytes never sent
while (drop > 0) {
byte[] first = chunks.peekFirst();
int avail = first.length - head;
int take = (int) Math.min(drop, avail);
head += take;
base += take;
length -= take;
drop -= take;
if (head == first.length) {
chunks.removeFirst();
head = 0;
}
}
}
/**
* The outstanding bytes at and after {@code off}, or {@code null} when
* {@code off} falls outside what is still held — which means the peer named
* an offset we can no longer satisfy and the stream cannot be resumed.
*/
Buffer from(long off) {
long skip = off - base;
if (skip < 0 || skip > length) return null;
Buffer out = Buffer.buffer((int) (length - skip));
int start = head;
for (byte[] chunk : chunks) {
int avail = chunk.length - start;
if (skip >= avail) {
skip -= avail;
start = 0;
continue;
}
out.appendBytes(chunk, start + (int) skip, avail - (int) skip);
skip = 0;
start = 0;
}
return out;
}
void clear() {
chunks.clear();
head = 0;
length = 0;
}
}
server/src/main/java/io/icybear/redapricot/WorkerConn.java
+207 -66
View File
@@ -4,15 +4,12 @@ import io.icybear.redapricot.net.EncryptedFrames;
import io.icybear.redapricot.util.ProtoReader;
import io.icybear.redapricot.util.ProtoWriter;
import io.vertx.core.buffer.Buffer;
import io.vertx.core.net.NetSocket;
import lombok.RequiredArgsConstructor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
/**
* An authenticated worker connection (Magic 0x02). Multiplexes many player
@@ -34,28 +31,17 @@ public final class WorkerConn {
private final String id;
private final int sendWndInit; // client's advertised per-stream receive window (our send budget)
private final int recvWndInit; // our advertised per-stream receive window (basis for credit grants)
/**
* Whether this conn negotiated stream resumption. Gates the park path: a
* client that will never reattach is better served by an immediate close than
* by a player left hanging for the whole grace period.
*/
private final boolean resume;
private final Map<Integer, StreamState> streams = new HashMap<>();
private final Map<Integer, PlayerStream> streams = new HashMap<>();
// Aggregate backpressure for the single shared worker socket: players parked
// until its write queue drains. Per-stream fairness is the credit windows'
// job; this only reacts to the whole pipe being congested.
private final Set<StreamState> upstreamPaused = new HashSet<>();
private boolean workerDrainArmed = false; // whether the worker socket's single drainHandler is set
/** Per-stream flow-control bookkeeping. */
private final class StreamState {
final NetSocket player;
int sendWnd = sendWndInit; // budget for player -> client DATA
Buffer pendingUp; // player bytes awaiting send window (player is paused meanwhile)
boolean pausedForWindow;
int credited; // client -> player bytes flushed but not yet granted back
StreamState(NetSocket player) {
this.player = player;
}
}
public void onFrame(byte[] payload) {
ProtoReader r = new ProtoReader(payload);
int type = r.readUByte();
@@ -64,6 +50,8 @@ public final class WorkerConn {
case Protocol.MUX_SYN -> handleSyn(sid, r.readBytes(Protocol.CID_LEN));
case Protocol.MUX_DATA -> handleData(sid, r.readBuffer(r.remaining()));
case Protocol.MUX_WND -> handleWnd(sid, r.readVarInt());
case Protocol.MUX_RESUME ->
handleResume(sid, r.readBytes(Protocol.CID_LEN), r.readI64(), r.readI64());
case Protocol.MUX_FIN, Protocol.MUX_RST -> closeStream(sid);
case Protocol.MUX_PING -> sendPong(r.readI64());
case Protocol.MUX_PONG -> { /* liveness only; arrival is what matters */ }
@@ -76,26 +64,40 @@ public final class WorkerConn {
PendingPlayer p = hub.takePending(cid);
if (p == null) {
LOG.warn("worker {} SYN for unknown CID", id);
sendRst(sid);
sendRst(sid, Protocol.RST_UNKNOWN_STREAM);
return;
}
NetSocket player = p.getSocket();
StreamState st = new StreamState(player);
PlayerStream st = new PlayerStream(p, this, sid, sendWndInit);
st.resumable = resume;
streams.put(sid, st);
hub.addStream(st);
// From now on the player socket belongs to this stream.
player.handler(buf -> {
sendUpstream(sid, st, buf);
checkAggregate(st);
});
player.closeHandler(v -> onPlayerGone(sid, st));
player.exceptionHandler(t -> onPlayerGone(sid, st));
// From now on the player socket belongs to this stream. The handlers are
// installed once and route through the hub, which dispatches to whichever
// conn currently carries the stream.
//
// They must not call this conn's methods directly: a lambda defined here
// captures `this`, so after the stream moves to another conn it would keep
// writing into the dead one's transport, where sends are silently dropped
// and the player goes mute with nothing logged. Re-installing handlers on
// every reattach would be the other option, but a Vert.x socket resumed
// while a stale handler is still attached loses bytes, so routing beats
// rebinding.
st.player.handler(buf -> hub.onPlayerData(st, buf));
st.player.closeHandler(v -> hub.onPlayerGone(st));
st.player.exceptionHandler(t -> hub.onPlayerGone(st));
// Forward the buffered handshake (and any pipelined bytes), then resume.
sendUpstream(sid, st, p.getBuffered());
if (!st.pausedForWindow) player.resume();
sendUpstream(st, p.getBuffered());
maybeResumePlayer(st);
checkAggregate(st);
LOG.info("worker {} stream {} bound to {}", id, sid, st.pattern);
}
/** Player bytes arrived on a stream this conn currently carries. */
void playerData(PlayerStream st, Buffer buf) {
sendUpstream(st, buf);
checkAggregate(st);
LOG.info("worker {} stream {} bound to {}", id, sid, p.getPattern());
}
/**
@@ -103,12 +105,12 @@ public final class WorkerConn {
* the overflow is parked in {@code pendingUp} and the player socket paused
* until the client grants more credit.
*/
private void sendUpstream(int sid, StreamState st, Buffer buf) {
private void sendUpstream(PlayerStream st, Buffer buf) {
if (st.pendingUp != null) { // still waiting for window; keep ordering
st.pendingUp.appendBuffer(buf);
return;
}
int off = drainUpstream(sid, st, buf, 0);
int off = drainUpstream(st, buf, 0);
if (off < buf.length()) {
st.pendingUp = buf.getBuffer(off, buf.length());
if (!st.pausedForWindow) {
@@ -119,10 +121,20 @@ public final class WorkerConn {
}
/** Send from {@code buf[off..]} within the stream window, chunked; returns the new offset. */
private int drainUpstream(int sid, StreamState st, Buffer buf, int off) {
private int drainUpstream(PlayerStream st, Buffer buf, int off) {
while (off < buf.length() && st.sendWnd > 0) {
int n = Math.min(Math.min(CHUNK, st.sendWnd), buf.length() - off);
sendData(sid, buf.getBytes(off, off + n));
byte[] chunk = buf.getBytes(off, off + n);
if (st.resumable) {
// Retain before sending. A frame written to a dying socket is
// lost with no notification, so the only trustworthy record of
// what the client still owes us is taken before the attempt.
// Retention is free here: the chunk was materialized to be sent.
st.unacked.advance(st.ackedOffset);
st.unacked.append(chunk);
}
st.sentOffset += n;
sendData(st.sid, chunk);
st.sendWnd -= n;
off += n;
}
@@ -131,51 +143,155 @@ public final class WorkerConn {
/** The client granted {@code delta} more bytes of credit on a stream. */
private void handleWnd(int sid, int delta) {
StreamState st = streams.get(sid);
PlayerStream st = streams.get(sid);
if (st == null || delta <= 0) return;
// The running sum doubles as the acked offset: the client grants credit
// exactly as bytes reach the destination socket, so a credited byte can
// never need retransmitting.
st.ackedOffset += delta;
st.sendWnd += delta;
if (st.pendingUp != null) {
Buffer pending = st.pendingUp;
int off = drainUpstream(sid, st, pending, 0);
int off = drainUpstream(st, pending, 0);
st.pendingUp = off >= pending.length() ? null : pending.getBuffer(off, pending.length());
}
if (st.pendingUp == null && st.pausedForWindow) {
st.pausedForWindow = false;
if (!upstreamPaused.contains(st)) st.player.resume();
}
if (st.pendingUp == null) st.pausedForWindow = false;
maybeResumePlayer(st);
checkAggregate(st);
}
private void handleData(int sid, Buffer data) {
StreamState st = streams.get(sid);
PlayerStream st = streams.get(sid);
if (st == null) return;
// Never pause the shared socket: the client bounds what it sends per
// stream to our advertised window, so a slow player only piles up a
// bounded amount in its own write queue; credit is granted back as the
// write completes (i.e. the bytes reached the player socket).
int len = data.length();
// Accepted the moment the bytes are taken off the wire, not when the write
// completes. Completion is asynchronous and suppressed once the connection
// closes — precisely when a reattach needs this number to be right — so
// reporting delivery would under-count and make the client replay bytes
// the player already has.
st.acceptedOffset += len;
st.player.write(data).onComplete(ar -> {
if (ar.failed() || frames.isClosed() || streams.get(sid) != st) return;
if (ar.failed()) return;
// Both counters advance even if this conn has since died or the stream
// has moved on. Discarding them would destroy up to half a window of
// credit per outage, and — worse — leave the delivered offset that a
// reattach restates the window from permanently short.
st.deliveredOffset += len;
st.credited += len;
if (st.worker != this || frames.isClosed()) return;
if (st.credited * 2 >= recvWndInit) {
int delta = st.credited;
st.credited = 0;
sendWnd(sid, delta);
sendWnd(st.sid, delta);
}
});
}
/**
* Reattach a parked stream to this connection (§7.5).
*
* <p>Runs to completion in one event-loop turn — rebind, acknowledge, replay —
* so the hub's single-threaded model makes the ordering race-free by
* construction, with no interleaving of live and replayed bytes to reason
* about.
*/
private void handleResume(int sid, byte[] cid, long clientAccepted, long clientDelivered) {
PlayerStream st = hub.takeParked(cid);
if (st == null) {
// Tell a stream we have never heard of apart from one that is still
// bound: the first is terminal for the client, the second means a
// racing attempt won and this one must leave the player alone.
boolean bound = hub.streamByCid(cid) != null;
LOG.warn("worker {} RESUME for {} CID", id, bound ? "still-bound" : "unknown");
sendRst(sid, bound ? Protocol.RST_ALREADY_BOUND : Protocol.RST_UNKNOWN_STREAM);
return;
}
// Delivery is a strictly stronger fact than credit — the client only
// credits what it has delivered — so the reported offset can be adopted
// wholesale. Doing so also repairs the ledger: the grants destroyed by the
// outage are exactly the gap between the two, and without this the
// retained region would carry that dead prefix for the stream's whole life.
st.ackedOffset = Math.max(st.ackedOffset, clientDelivered);
st.unacked.advance(st.ackedOffset);
Buffer replay = st.unacked.from(clientAccepted);
if (replay == null) {
LOG.warn("worker {} RESUME at offset {} outside the retained region [{}, {}]; closing player",
id, clientAccepted, st.unacked.base(), st.unacked.end());
sendRst(sid, Protocol.RST_UNKNOWN_STREAM);
hub.removeStream(st);
st.player.close();
return;
}
st.worker = this;
st.sid = sid;
st.resumable = resume;
streams.put(sid, st);
// Restate the window rather than patching it. Three offsets, three jobs:
// the replay above is measured from what the client *accepted*, the window
// from what it *delivered* (the window being a promise about undelivered
// bytes), and never from what it *credited* — credit travels as deltas, and
// the grants in flight when the connection died are gone for good, so a
// window derived from them stays permanently short. When a full window was
// outstanding at the drop that means permanently zero, which deadlocks:
// nothing can be sent, so no credit can ever come back.
int outstanding = st.unacked.length();
st.sendWnd = Math.max(0, sendWndInit - outstanding);
// Symmetrically, drop our own pending credit instead of flushing it: the
// delivered offset in the ack already carries everything those deltas
// would have, and sending both would grant the same bytes twice.
st.credited = 0;
// A fresh capability per reattach keeps a CID single-use, so a leaked one
// never grants more than the outage it was observed in.
byte[] newCid = hub.newCid();
hub.rekeyStream(st, newCid);
frames.send(new ProtoWriter()
.u8(Protocol.MUX_RESUME_ACK).varInt(sid)
.i64(st.acceptedOffset)
.i64(st.deliveredOffset)
.bytes(newCid)
.toBytes());
// Replayed straight to the wire: it must not be re-charged against the
// window or re-appended to the retained region, both of which
// drainUpstream would do.
for (int off = 0; off < replay.length(); off += CHUNK) {
int end = Math.min(off + CHUNK, replay.length());
sendData(sid, replay.getBytes(off, end));
}
if (st.pendingUp != null) {
Buffer pending = st.pendingUp;
int off = drainUpstream(st, pending, 0);
st.pendingUp = off >= pending.length() ? null : pending.getBuffer(off, pending.length());
}
if (st.pendingUp == null) st.pausedForWindow = false;
maybeResumePlayer(st);
checkAggregate(st);
LOG.info("worker {} stream {} resumed ({} bytes replayed, {} outstanding)",
id, sid, replay.length(), outstanding);
}
private void closeStream(int sid) {
StreamState st = streams.remove(sid);
PlayerStream st = streams.remove(sid);
if (st != null) {
upstreamPaused.remove(st);
st.worker = null;
hub.removeStream(st);
st.player.close();
}
}
/** Park the player if the shared worker socket's write queue is congested. */
private void checkAggregate(StreamState st) {
if (frames.writeQueueFull() && upstreamPaused.add(st)) {
private void checkAggregate(PlayerStream st) {
if (!st.pausedForAggregate && frames.writeQueueFull()) {
st.pausedForAggregate = true;
st.player.pause();
armWorkerDrain();
}
@@ -187,20 +303,30 @@ public final class WorkerConn {
workerDrainArmed = true;
frames.socket().drainHandler(v -> {
workerDrainArmed = false;
if (upstreamPaused.isEmpty()) return;
StreamState[] parked = upstreamPaused.toArray(new StreamState[0]);
upstreamPaused.clear();
for (StreamState st : parked) {
if (!st.pausedForWindow) st.player.resume();
for (PlayerStream st : streams.values()) {
if (!st.pausedForAggregate) continue;
st.pausedForAggregate = false;
maybeResumePlayer(st);
}
});
}
/** The player side of a stream vanished: drop it from every table, release any backpressure it held, and FIN the peer if still live. */
private void onPlayerGone(int sid, StreamState st) {
boolean wasLive = streams.remove(sid) == st;
upstreamPaused.remove(st);
if (wasLive) sendFin(sid);
/**
* Resume the player socket if no reason to hold it applies any more.
*
* <p>The single arbitration point for every pause reason. Vert.x
* {@code pause()} is a flag rather than a counter, so resuming while another
* reason still holds would let bytes through that we have nowhere to put.
*/
private void maybeResumePlayer(PlayerStream st) {
if (st.shouldFlow()) st.player.resume();
}
/** The player side of a stream this conn carries vanished: unbind it and FIN the client. */
void playerGone(PlayerStream st) {
boolean wasLive = streams.remove(st.sid) == st;
st.worker = null;
if (wasLive) sendFin(st.sid);
}
private void sendData(int sid, byte[] data) {
@@ -211,8 +337,9 @@ public final class WorkerConn {
frames.send(new ProtoWriter().u8(Protocol.MUX_FIN).varInt(sid).toBytes());
}
private void sendRst(int sid) {
frames.send(new ProtoWriter().u8(Protocol.MUX_RST).varInt(sid).toBytes());
/** The reason is a trailing byte, optional on the wire; peers that predate it send none. */
private void sendRst(int sid, int reason) {
frames.send(new ProtoWriter().u8(Protocol.MUX_RST).varInt(sid).u8(reason).toBytes());
}
private void sendWnd(int sid, int delta) {
@@ -224,10 +351,24 @@ public final class WorkerConn {
frames.send(new ProtoWriter().u8(Protocol.MUX_PONG).varInt(Protocol.MUX_CTL_SID).i64(nonce).toBytes());
}
/**
* Only the tunnel leg died. Where the session negotiated resumption the
* player sockets are hung rather than closed, and wait for the client to
* reattach their streams over a fresh conn (§7.5); otherwise this is the old,
* unconditional close.
*/
public void onClose() {
for (StreamState st : streams.values()) st.player.close();
int parked = 0;
for (PlayerStream st : streams.values()) {
st.worker = null;
if (hub.park(st)) {
parked++;
} else {
hub.removeStream(st);
st.player.close();
}
}
LOG.info("worker {} closed ({} of {} player(s) hung for reattach)", id, parked, streams.size());
streams.clear();
upstreamPaused.clear();
LOG.info("worker {} closed", id);
}
}
server/src/test/java/io/icybear/redapricot/CryptoCodecTest.java
+2 -1
View File
@@ -78,7 +78,8 @@ class CryptoCodecTest {
/** A Hub whose event loop is never touched (register/match/normalize use no Vert.x state). */
private static Hub testHub() {
return new Hub(null, new Config("0.0.0.0", 25565, "test-psk", 30_000L, 10_000L,
Protocol.DEFAULT_STREAM_WINDOW, 90_000L));
Protocol.DEFAULT_STREAM_WINDOW, 90_000L,
true, 20_000L, 256, 256L * 2 * Protocol.DEFAULT_STREAM_WINDOW, 0L, 15_000L));
}
private static ControlSession testSession(Hub hub, String id) {