break: replace muxed workers with 1:1 tunnels
Worker frames are now FrameType + payload; there is no stream id. Each player gets its own worker conn. maxTunnels (default 256) caps concurrent tunnels. The old maxConn pool size is ignored so existing configs do not silently admit only a handful of players. Resume, per-direction windows, the control session, and the DATA-only shaper stay. A dropped worker still hangs that one player and reattaches over a fresh conn. Add a hub-side per-IP limiter for player intents only (default 8/s, burst 16, 64 concurrent). Unmatched hostnames consume a token; Intent 17 is never counted. 0 disables each knob.
This commit is contained in:
+172
-291
@@ -12,166 +12,87 @@ import (
|
||||
"github.com/iceBear67/redapricot/client/wire"
|
||||
)
|
||||
|
||||
// errPoolClosed is returned by Allocate after Close: the pool is shutting down
|
||||
// errPoolClosed is returned by Dial after Close: the set is shutting down
|
||||
// and must not start new dials, so a caller (handleControlRequest,
|
||||
// allocateForResume) gives up rather than wait on a cond no one will satisfy.
|
||||
var errPoolClosed = errors.New("worker pool closed")
|
||||
// allocateForResume) gives up rather than waiting on a hub that will never
|
||||
// be used.
|
||||
var errPoolClosed = errors.New("worker set closed")
|
||||
|
||||
// StreamsBeforeGrowing is how many streams a worker conn may carry before the
|
||||
// pool starts opening another one. Set to 1 so the pool fans out to maxConn
|
||||
// under load *before* stacking streams: concentrating every player on a single
|
||||
// TCP connection makes that connection a shared point of failure, which is
|
||||
// exactly how a whole server's worth of players used to drop at once.
|
||||
const StreamsBeforeGrowing = 1
|
||||
// errTooManyTunnels is returned when live+in-flight worker conns already
|
||||
// equal maxTunnels. The ControlRequest is dropped; the hub closes the player
|
||||
// when pendingTimeoutMs fires.
|
||||
var errTooManyTunnels = errors.New("maxTunnels reached")
|
||||
|
||||
// allocateAttempts bounds how many times a caller retries Allocate when the
|
||||
// conn it was handed dies before the stream could be registered on it. The race
|
||||
// is narrow and each retry picks a different conn, so a small bound is enough;
|
||||
// an unbounded loop would spin against a hub that is refusing every connection.
|
||||
const allocateAttempts = 3
|
||||
// dialAttempts bounds how many times a caller retries Dial when the conn it
|
||||
// was handed dies before the stream could be attached to it. The race is
|
||||
// narrow; an unbounded loop would spin against a hub that is refusing every
|
||||
// connection.
|
||||
const dialAttempts = 3
|
||||
|
||||
// WorkerPool manages up to maxConn worker connections and allocates streams
|
||||
// using the least-loaded strategy (PROTOCOL.md §7.1).
|
||||
// WorkerPool tracks live 1:1 worker connections up to maxTunnels
|
||||
// (PROTOCOL.md §7.1). There is no least-loaded placement and no sharing:
|
||||
// every player gets its own TCP connection.
|
||||
type WorkerPool struct {
|
||||
client *Client
|
||||
maxConn int
|
||||
client *Client
|
||||
maxTunnels int
|
||||
|
||||
connSeq atomic.Int64 // conn ids, for log correlation
|
||||
|
||||
mu sync.Mutex
|
||||
cond *sync.Cond
|
||||
conns []*WorkerConn
|
||||
dialing int // dials currently in flight (foreground + background)
|
||||
dialGen uint64
|
||||
dialErr error // most recent dial failure
|
||||
closed bool // closeAll ran; no new conns may join the pool
|
||||
conns map[*WorkerConn]struct{}
|
||||
dialing int // dials currently in flight
|
||||
closed bool // closeAll ran; no new conns may join
|
||||
}
|
||||
|
||||
func newWorkerPool(c *Client, maxConn int) *WorkerPool {
|
||||
p := &WorkerPool{client: c, maxConn: maxConn}
|
||||
p.cond = sync.NewCond(&p.mu)
|
||||
return p
|
||||
func newWorkerPool(c *Client, maxTunnels int) *WorkerPool {
|
||||
return &WorkerPool{
|
||||
client: c,
|
||||
maxTunnels: maxTunnels,
|
||||
conns: make(map[*WorkerConn]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// Allocate returns a worker conn and a fresh stream id to place a new stream on.
|
||||
// Dial opens a dedicated worker conn for one player.
|
||||
//
|
||||
// A dial is never performed while holding p.mu: session establishment involves
|
||||
// network I/O, and holding the pool lock across it would park every other
|
||||
// player behind one unresponsive hub. When the pool is empty exactly one caller
|
||||
// dials and the rest wait on the condition variable; when the pool is merely
|
||||
// below maxConn, growth happens in the background and the caller is served
|
||||
// immediately by an existing conn.
|
||||
func (p *WorkerPool) Allocate() (*WorkerConn, int, error) {
|
||||
// network I/O, and holding the lock across it would park every other player
|
||||
// behind one unresponsive hub. Each caller dials independently; unlike the
|
||||
// old mux pool there is no shared conn to wait for.
|
||||
func (p *WorkerPool) Dial() (*WorkerConn, error) {
|
||||
p.mu.Lock()
|
||||
for {
|
||||
if p.closed {
|
||||
// Close won. No new conn may join the pool, so no stream may be
|
||||
// placed; waiting on the cond could only be satisfied by a dial we
|
||||
// must not start.
|
||||
p.mu.Unlock()
|
||||
return nil, 0, errPoolClosed
|
||||
}
|
||||
best, bestCount := p.leastLoadedLocked()
|
||||
if best != nil {
|
||||
p.maybeGrowLocked(bestCount)
|
||||
p.mu.Unlock()
|
||||
return best, best.newSid(), nil
|
||||
}
|
||||
if p.dialing > 0 {
|
||||
// Someone is already dialing the first conn; wait for it rather
|
||||
// than piling up redundant connections.
|
||||
gen := p.dialGen
|
||||
p.cond.Wait()
|
||||
if len(p.conns) == 0 && p.dialGen != gen && p.dialErr != nil {
|
||||
err := p.dialErr
|
||||
p.mu.Unlock()
|
||||
return nil, 0, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
p.dialing++
|
||||
p.mu.Unlock()
|
||||
wc, err := p.dialWorker()
|
||||
p.mu.Lock()
|
||||
p.dialing--
|
||||
p.dialGen++
|
||||
p.dialErr = err
|
||||
if err != nil {
|
||||
p.cond.Broadcast()
|
||||
p.mu.Unlock()
|
||||
return nil, 0, err
|
||||
}
|
||||
if p.closed {
|
||||
// Close raced this dial: the conn must not enter the pool. Closing
|
||||
// it here, under p.mu, is a raw socket close — fine, and it makes
|
||||
// the shutdown atomic with the pool state.
|
||||
p.cond.Broadcast()
|
||||
p.mu.Unlock()
|
||||
_ = wc.fc.Close()
|
||||
return nil, 0, errPoolClosed
|
||||
}
|
||||
p.conns = append(p.conns, wc)
|
||||
p.cond.Broadcast()
|
||||
}
|
||||
}
|
||||
|
||||
// leastLoadedLocked returns the worker conn carrying the fewest streams.
|
||||
func (p *WorkerPool) leastLoadedLocked() (*WorkerConn, int) {
|
||||
var best *WorkerConn
|
||||
bestCount := 0
|
||||
for _, wc := range p.conns {
|
||||
n := wc.streamCount()
|
||||
if best == nil || n < bestCount {
|
||||
best = wc
|
||||
bestCount = n
|
||||
}
|
||||
}
|
||||
return best, bestCount
|
||||
}
|
||||
|
||||
// maybeGrowLocked opens one more worker conn in the background when the pool is
|
||||
// below maxConn and the least-loaded conn is already carrying streams. The
|
||||
// caller does not wait for it: it keeps using the conn it already has, and the
|
||||
// new one picks up subsequent players.
|
||||
func (p *WorkerPool) maybeGrowLocked(bestCount int) {
|
||||
if bestCount < StreamsBeforeGrowing {
|
||||
return
|
||||
}
|
||||
if p.closed {
|
||||
return // shutdown; do not start dials nobody will join the pool
|
||||
p.mu.Unlock()
|
||||
return nil, errPoolClosed
|
||||
}
|
||||
if len(p.conns)+p.dialing >= p.maxConn {
|
||||
if bestCount > SaturationThreshold {
|
||||
log.Printf("worker pool at maxConn=%d with %d streams on the least-loaded conn", p.maxConn, bestCount)
|
||||
}
|
||||
return
|
||||
if len(p.conns)+p.dialing >= p.maxTunnels {
|
||||
p.mu.Unlock()
|
||||
return nil, errTooManyTunnels
|
||||
}
|
||||
p.dialing++
|
||||
go func() {
|
||||
wc, err := p.dialWorker()
|
||||
var surplus *WorkerConn
|
||||
p.mu.Lock()
|
||||
p.dialing--
|
||||
p.dialGen++
|
||||
p.dialErr = err
|
||||
switch {
|
||||
case err != nil:
|
||||
case p.closed:
|
||||
surplus = wc // Close raced this background dial
|
||||
case len(p.conns) < p.maxConn:
|
||||
p.conns = append(p.conns, wc)
|
||||
default:
|
||||
surplus = wc // raced with another dial
|
||||
}
|
||||
p.cond.Broadcast()
|
||||
p.mu.Unlock()
|
||||
|
||||
wc, err := p.dialWorker()
|
||||
|
||||
p.mu.Lock()
|
||||
p.dialing--
|
||||
if err != nil {
|
||||
p.mu.Unlock()
|
||||
if err != nil {
|
||||
log.Printf("worker pool: background dial failed: %v", err)
|
||||
}
|
||||
if surplus != nil {
|
||||
_ = surplus.fc.Close()
|
||||
}
|
||||
}()
|
||||
return nil, err
|
||||
}
|
||||
if p.closed {
|
||||
// Close raced this dial: the conn must not enter the set.
|
||||
p.mu.Unlock()
|
||||
_ = wc.fc.Close()
|
||||
return nil, errPoolClosed
|
||||
}
|
||||
if len(p.conns) >= p.maxTunnels {
|
||||
p.mu.Unlock()
|
||||
_ = wc.fc.Close()
|
||||
return nil, errTooManyTunnels
|
||||
}
|
||||
p.conns[wc] = struct{}{}
|
||||
p.mu.Unlock()
|
||||
return wc, nil
|
||||
}
|
||||
|
||||
// dialWorker establishes one worker conn. It must be called without p.mu held.
|
||||
@@ -188,8 +109,6 @@ func (p *WorkerPool) dialWorker() (*WorkerConn, error) {
|
||||
resume: sess.resume,
|
||||
grace: p.client.resumeGrace(sess.hubGrace),
|
||||
id: int(p.connSeq.Add(1)),
|
||||
streams: make(map[int]*Stream),
|
||||
nextSid: 1,
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
if p.client.statsOn() {
|
||||
@@ -200,7 +119,7 @@ func (p *WorkerPool) dialWorker() (*WorkerConn, error) {
|
||||
if sess.heartbeat {
|
||||
go wc.heartbeatLoop(p.client.cfg.pingInterval(), p.client.cfg.heartbeatTimeout())
|
||||
} else {
|
||||
log.Printf("worker conn: hub does not support the mux heartbeat; " +
|
||||
log.Printf("worker conn: hub does not support the worker heartbeat; " +
|
||||
"a silently dropped path will only be caught by TCP keepalive")
|
||||
}
|
||||
log.Printf("opened worker conn (send window %d, recv window %d, heartbeat %v, resume %v)",
|
||||
@@ -217,41 +136,32 @@ func (p *WorkerPool) count() int {
|
||||
func (p *WorkerPool) remove(wc *WorkerConn) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
for i, c := range p.conns {
|
||||
if c == wc {
|
||||
p.conns = append(p.conns[:i], p.conns[i+1:]...)
|
||||
// A waiter parked on an empty pool must re-evaluate: it may now
|
||||
// need to dial rather than keep waiting for this conn.
|
||||
p.cond.Broadcast()
|
||||
return
|
||||
}
|
||||
}
|
||||
delete(p.conns, wc)
|
||||
}
|
||||
|
||||
func (p *WorkerPool) closeAll() {
|
||||
p.mu.Lock()
|
||||
p.closed = true
|
||||
conns := append([]*WorkerConn(nil), p.conns...)
|
||||
// Wake waiters parked in Allocate: the closed flag they re-check is the
|
||||
// only thing that can release them now that no conn will ever join.
|
||||
p.cond.Broadcast()
|
||||
conns := make([]*WorkerConn, 0, len(p.conns))
|
||||
for wc := range p.conns {
|
||||
conns = append(conns, wc)
|
||||
}
|
||||
p.mu.Unlock()
|
||||
for _, wc := range conns {
|
||||
_ = wc.fc.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// WorkerConn is one multiplexed worker connection to the hub.
|
||||
//
|
||||
// Only genuinely per-connection state lives here. Client-wide values (the
|
||||
// shaper, the advertised receive window, the DATA chunk cap) belong to Client:
|
||||
// reading them through a connection pointer would make every such read a
|
||||
// re-parenting hazard once a stream can migrate between conns.
|
||||
// WorkerConn is one 1:1 worker connection to the hub: it carries exactly one
|
||||
// player. Only genuinely per-connection state lives here. Client-wide values
|
||||
// (the shaper, the advertised receive window, the DATA chunk cap) belong to
|
||||
// Client: reading them through a connection pointer would make every such
|
||||
// read a re-parenting hazard once a stream can migrate between conns.
|
||||
type WorkerConn struct {
|
||||
pool *WorkerPool
|
||||
fc *wire.FramedConn
|
||||
|
||||
sendWndInit int // hub's advertised per-stream receive window (our send budget)
|
||||
sendWndInit int // hub's advertised receive window (our send budget)
|
||||
|
||||
// resume is whether this conn negotiated stream resumption, and grace how
|
||||
// long a stream parked from it may keep trying to reattach. Both are
|
||||
@@ -266,17 +176,16 @@ type WorkerConn struct {
|
||||
id int // for log correlation only
|
||||
stats *connStats // nil unless diagnostics are enabled
|
||||
|
||||
mu sync.Mutex
|
||||
streams map[int]*Stream
|
||||
nextSid int
|
||||
closed bool // readLoop has exited; registerStream must refuse
|
||||
mu sync.Mutex
|
||||
stream *Stream
|
||||
closed bool // readLoop has exited; attach must refuse
|
||||
}
|
||||
|
||||
// heartbeatLoop proves the worker conn is still carrying frames end to end. TCP
|
||||
// alone cannot tell us: a middlebox that drops an established flow (conntrack
|
||||
// expiry, firewall state loss) sends no FIN or RST, so the read loop would park
|
||||
// forever, the dead conn would stay in the pool, and every player routed to it
|
||||
// would silently fail until the process restarted.
|
||||
// forever and the player on this conn would silently fail until the process
|
||||
// restarted.
|
||||
func (wc *WorkerConn) heartbeatLoop(interval, timeout time.Duration) {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
@@ -286,12 +195,11 @@ func (wc *WorkerConn) heartbeatLoop(interval, timeout time.Duration) {
|
||||
return
|
||||
case <-ticker.C:
|
||||
if silent := time.Since(time.UnixMilli(wc.lastPong.Load())); silent > timeout {
|
||||
log.Printf("worker conn silent for %s; dropping it and its %d stream(s)",
|
||||
silent.Round(time.Second), wc.streamCount())
|
||||
log.Printf("worker conn silent for %s; dropping it", silent.Round(time.Second))
|
||||
_ = wc.fc.Close() // readLoop unblocks and tears everything down
|
||||
return
|
||||
}
|
||||
msg := wire.NewWriter().U8(MuxPing).VarInt(MuxCtlSid).I64(time.Now().UnixMilli()).Out()
|
||||
msg := wire.NewWriter().U8(MuxPing).I64(time.Now().UnixMilli()).Out()
|
||||
if err := wc.fc.WriteFrame(msg); err != nil {
|
||||
return
|
||||
}
|
||||
@@ -299,56 +207,39 @@ func (wc *WorkerConn) heartbeatLoop(interval, timeout time.Duration) {
|
||||
}
|
||||
}
|
||||
|
||||
func (wc *WorkerConn) streamCount() int {
|
||||
wc.mu.Lock()
|
||||
defer wc.mu.Unlock()
|
||||
return len(wc.streams)
|
||||
}
|
||||
|
||||
func (wc *WorkerConn) newSid() int {
|
||||
wc.mu.Lock()
|
||||
defer wc.mu.Unlock()
|
||||
sid := wc.nextSid
|
||||
wc.nextSid++
|
||||
return sid
|
||||
}
|
||||
|
||||
// registerStream publishes a stream in the conn's table, or reports false if
|
||||
// the conn has already died.
|
||||
// attach publishes a stream on this conn, or reports false if the conn has
|
||||
// already died (or is already bound — which would be a caller bug).
|
||||
//
|
||||
// The check is not advisory. Allocate hands out a (conn, sid) pair under the
|
||||
// pool lock, and the conn's readLoop can exit before the caller gets here — it
|
||||
// has then already swapped the stream map, so a blind insert would land in a map
|
||||
// nothing iterates and the stream would never be torn down. That normally hides
|
||||
// behind a failing SYN, but not on a half-open conn whose readLoop died on a
|
||||
// framing error while the socket is still writable. Callers must re-Allocate.
|
||||
func (wc *WorkerConn) registerStream(sid int, st *Stream) bool {
|
||||
// The check is not advisory. Dial hands out a conn, and the conn's readLoop
|
||||
// can exit before the caller gets here. A blind store would land on a conn
|
||||
// nothing iterates and the stream would never be torn down.
|
||||
func (wc *WorkerConn) attach(st *Stream) bool {
|
||||
wc.mu.Lock()
|
||||
defer wc.mu.Unlock()
|
||||
if wc.closed {
|
||||
if wc.closed || wc.stream != nil {
|
||||
return false
|
||||
}
|
||||
wc.streams[sid] = st
|
||||
wc.stream = st
|
||||
return true
|
||||
}
|
||||
|
||||
func (wc *WorkerConn) getStream(sid int) *Stream {
|
||||
func (wc *WorkerConn) getStream() *Stream {
|
||||
wc.mu.Lock()
|
||||
defer wc.mu.Unlock()
|
||||
return wc.streams[sid]
|
||||
return wc.stream
|
||||
}
|
||||
|
||||
func (wc *WorkerConn) removeStream(sid int) *Stream {
|
||||
func (wc *WorkerConn) detach() *Stream {
|
||||
wc.mu.Lock()
|
||||
defer wc.mu.Unlock()
|
||||
st := wc.streams[sid]
|
||||
delete(wc.streams, sid)
|
||||
st := wc.stream
|
||||
wc.stream = nil
|
||||
return st
|
||||
}
|
||||
|
||||
// readLoop dispatches inbound mux frames. It must never block on a stream's
|
||||
// destination: DATA is only enqueued (the per-stream writeLoop does the actual
|
||||
// destination writes), so one slow destination cannot stall other streams.
|
||||
// readLoop dispatches inbound tunnel frames. DATA is only enqueued (the
|
||||
// stream's writeLoop does the actual destination writes), so a stalled
|
||||
// destination cannot stall liveness / WND / FIN dispatch on this conn.
|
||||
func (wc *WorkerConn) readLoop() {
|
||||
for {
|
||||
payload, err := wc.fc.ReadFrame()
|
||||
@@ -363,24 +254,20 @@ func (wc *WorkerConn) readLoop() {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
sid, err := r.VarInt()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
switch ftype {
|
||||
case MuxData:
|
||||
if st := wc.getStream(sid); st != nil {
|
||||
if st := wc.getStream(); st != nil {
|
||||
st.deliverFromHub(r.Remaining())
|
||||
}
|
||||
case MuxWnd:
|
||||
if delta, err := r.VarInt(); err == nil && delta > 0 {
|
||||
if st := wc.getStream(sid); st != nil {
|
||||
if st := wc.getStream(); st != nil {
|
||||
st.grantSendWnd(delta)
|
||||
}
|
||||
}
|
||||
case MuxFin:
|
||||
// Graceful: drain what is already queued to the destination first.
|
||||
if st := wc.removeStream(sid); st != nil {
|
||||
if st := wc.detach(); st != nil {
|
||||
st.gracefulFin()
|
||||
}
|
||||
case MuxRst:
|
||||
@@ -389,7 +276,7 @@ func (wc *WorkerConn) readLoop() {
|
||||
if b, err := r.U8(); err == nil {
|
||||
reason = int(b)
|
||||
}
|
||||
if st := wc.removeStream(sid); st != nil {
|
||||
if st := wc.detach(); st != nil {
|
||||
st.onRst(reason)
|
||||
}
|
||||
case MuxResumeAck:
|
||||
@@ -399,12 +286,12 @@ func (wc *WorkerConn) readLoop() {
|
||||
if aerr != nil || derr != nil || cerr != nil {
|
||||
continue
|
||||
}
|
||||
if st := wc.getStream(sid); st != nil {
|
||||
if st := wc.getStream(); st != nil {
|
||||
st.deliverResume(resumeResult{accepted: accepted, delivered: delivered, cid: cid})
|
||||
}
|
||||
case MuxPing:
|
||||
nonce, _ := r.I64()
|
||||
_ = wc.fc.WriteFrame(wire.NewWriter().U8(MuxPong).VarInt(MuxCtlSid).I64(nonce).Out())
|
||||
_ = wc.fc.WriteFrame(wire.NewWriter().U8(MuxPong).I64(nonce).Out())
|
||||
case MuxPong:
|
||||
now := time.Now()
|
||||
wc.lastPong.Store(now.UnixMilli())
|
||||
@@ -416,46 +303,42 @@ func (wc *WorkerConn) readLoop() {
|
||||
}
|
||||
}
|
||||
default:
|
||||
log.Printf("worker: unknown mux type %d", ftype)
|
||||
log.Printf("worker: unknown frame type %d", ftype)
|
||||
}
|
||||
}
|
||||
// Connection lost: tear down all streams and drop from pool.
|
||||
// Connection lost: tear down the bound stream and drop from the set.
|
||||
close(wc.done)
|
||||
wc.pool.remove(wc)
|
||||
wc.mu.Lock()
|
||||
// Marked before the map is swapped, under the same lock, so a concurrent
|
||||
// registerStream either lands in the map we are about to drain or is refused.
|
||||
// Marked before the pointer is cleared, under the same lock, so a concurrent
|
||||
// attach either lands in the field we are about to drain or is refused.
|
||||
wc.closed = true
|
||||
streams := make([]*Stream, 0, len(wc.streams))
|
||||
for _, st := range wc.streams {
|
||||
streams = append(streams, st)
|
||||
}
|
||||
wc.streams = make(map[int]*Stream)
|
||||
st := wc.stream
|
||||
wc.stream = nil
|
||||
wc.mu.Unlock()
|
||||
if st == nil {
|
||||
return
|
||||
}
|
||||
// Only the tunnel leg died. Where the session negotiated resumption the
|
||||
// destination sockets are kept open and each stream reattaches over a fresh
|
||||
// destination socket is kept open and the stream reattaches over a fresh
|
||||
// conn (§7.5); otherwise this is the old, unconditional teardown.
|
||||
// During Close there is no reattach to come: a stream that parked now would
|
||||
// hold its destination socket open past shutdown, so teardown instead.
|
||||
if wc.pool.client.closing.Load() {
|
||||
for _, st := range streams {
|
||||
st.teardown(false)
|
||||
}
|
||||
st.teardown(false)
|
||||
return
|
||||
}
|
||||
for _, st := range streams {
|
||||
if !st.park(wc.grace) {
|
||||
st.teardown(false)
|
||||
}
|
||||
if !st.park(wc.grace) {
|
||||
st.teardown(false)
|
||||
}
|
||||
}
|
||||
|
||||
func (wc *WorkerConn) sendSyn(sid int, cid []byte) error {
|
||||
return wc.fc.WriteFrame(wire.NewWriter().U8(MuxSyn).VarInt(sid).Bytes(cid).Out())
|
||||
func (wc *WorkerConn) sendSyn(cid []byte) error {
|
||||
return wc.fc.WriteFrame(wire.NewWriter().U8(MuxSyn).Bytes(cid).Out())
|
||||
}
|
||||
|
||||
func (wc *WorkerConn) sendData(sid int, data []byte) error {
|
||||
err := wc.fc.WriteFrame(wire.NewWriter().U8(MuxData).VarInt(sid).Bytes(data).Out())
|
||||
func (wc *WorkerConn) sendData(data []byte) error {
|
||||
err := wc.fc.WriteFrame(wire.NewWriter().U8(MuxData).Bytes(data).Out())
|
||||
if wc.stats != nil {
|
||||
wc.stats.framesOut.Add(1)
|
||||
if err != nil {
|
||||
@@ -465,42 +348,27 @@ func (wc *WorkerConn) sendData(sid int, data []byte) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (wc *WorkerConn) sendFin(sid int) {
|
||||
_ = wc.fc.WriteFrame(wire.NewWriter().U8(MuxFin).VarInt(sid).Out())
|
||||
func (wc *WorkerConn) sendFin() {
|
||||
_ = wc.fc.WriteFrame(wire.NewWriter().U8(MuxFin).Out())
|
||||
}
|
||||
|
||||
func (wc *WorkerConn) sendRst(sid int) {
|
||||
_ = wc.fc.WriteFrame(wire.NewWriter().U8(MuxRst).VarInt(sid).Out())
|
||||
func (wc *WorkerConn) sendRst() {
|
||||
_ = wc.fc.WriteFrame(wire.NewWriter().U8(MuxRst).Out())
|
||||
}
|
||||
|
||||
func (wc *WorkerConn) sendWndUpdate(sid, delta int) {
|
||||
_ = wc.fc.WriteFrame(wire.NewWriter().U8(MuxWnd).VarInt(sid).VarInt(delta).Out())
|
||||
func (wc *WorkerConn) sendWndUpdate(delta int) {
|
||||
_ = wc.fc.WriteFrame(wire.NewWriter().U8(MuxWnd).VarInt(delta).Out())
|
||||
}
|
||||
|
||||
// leg binds a stream to one worker conn. The two fields are only meaningful
|
||||
// together: stream ids are per-conn and restart at 1, so conn A's sid 3 and
|
||||
// conn B's sid 3 belong to different players. A torn read across the two would
|
||||
// credit, reset or FIN a stranger's stream, so the pair is swapped as one
|
||||
// immutable value rather than as two fields.
|
||||
type leg struct {
|
||||
wc *WorkerConn
|
||||
sid int
|
||||
}
|
||||
|
||||
// String formats one (conn, sid) snapshot for log correlation. Stream ids
|
||||
// restart at 1 per conn, so a bare sid cannot be traced across a reattach —
|
||||
// the conn id is what ties the log lines together.
|
||||
func (lg *leg) String() string { return fmt.Sprintf("conn%d/sid%d", lg.wc.id, lg.sid) }
|
||||
|
||||
// Stream bridges one player (via the hub) to one destination connection.
|
||||
//
|
||||
// Data from the hub is queued and written to the destination by a dedicated
|
||||
// writeLoop goroutine. The queue is bounded by the advertised receive window —
|
||||
// the hub never sends more un-credited bytes, so overflow is a protocol
|
||||
// violation and resets the stream.
|
||||
// violation and resets the tunnel.
|
||||
type Stream struct {
|
||||
client *Client
|
||||
leg atomic.Pointer[leg]
|
||||
client *Client
|
||||
wc atomic.Pointer[WorkerConn]
|
||||
mapping Mapping
|
||||
srcIP string
|
||||
srcPort int
|
||||
@@ -527,8 +395,8 @@ type Stream struct {
|
||||
// never interleave with live ones.
|
||||
//
|
||||
// Deliberately not s.mu: deliverFromHub takes s.mu from the worker readLoop,
|
||||
// and holding s.mu across a WriteFrame would stall frame dispatch for every
|
||||
// other stream on the connection.
|
||||
// and holding s.mu across a WriteFrame would stall frame dispatch (WND/FIN
|
||||
// /heartbeat replies) on this conn.
|
||||
sendMu sync.Mutex
|
||||
un unackedBuf // guarded by sendMu
|
||||
|
||||
@@ -578,10 +446,10 @@ type qentry struct {
|
||||
fromHub bool
|
||||
}
|
||||
|
||||
func newStream(c *Client, wc *WorkerConn, sid int, cid []byte, m Mapping, ip string, port int) *Stream {
|
||||
func newStream(c *Client, wc *WorkerConn, cid []byte, m Mapping, ip string, port int) *Stream {
|
||||
s := &Stream{client: c, cid: cid, mapping: m, srcIP: ip, srcPort: port,
|
||||
resumable: wc.resume, sendWnd: wc.sendWndInit, done: make(chan struct{})}
|
||||
s.leg.Store(&leg{wc: wc, sid: sid})
|
||||
s.wc.Store(wc)
|
||||
if c.statsOn() {
|
||||
s.stats = &streamStats{opened: time.Now()}
|
||||
}
|
||||
@@ -592,20 +460,28 @@ func newStream(c *Client, wc *WorkerConn, sid int, cid []byte, m Mapping, ip str
|
||||
return s
|
||||
}
|
||||
|
||||
// conn returns the stream's current binding. Every caller must take exactly one
|
||||
// snapshot and use both fields from it; re-loading mid-operation reintroduces
|
||||
// the torn-pair hazard the leg exists to prevent.
|
||||
func (s *Stream) conn() *leg { return s.leg.Load() }
|
||||
// conn returns the stream's current worker conn. May be the original or a
|
||||
// reattach; callers that send must take one snapshot and use it for the
|
||||
// whole operation so a concurrent rebind cannot split a write across conns.
|
||||
func (s *Stream) conn() *WorkerConn { return s.wc.Load() }
|
||||
|
||||
func (s *Stream) name() string {
|
||||
if wc := s.conn(); wc != nil {
|
||||
return fmt.Sprintf("conn%d", wc.id)
|
||||
}
|
||||
return "conn?"
|
||||
}
|
||||
|
||||
// run dials the destination, optionally writes the PROXY v2 header, then pumps
|
||||
// destination -> hub (respecting the stream send window when negotiated).
|
||||
// destination -> hub (respecting the send window when negotiated).
|
||||
func (s *Stream) run() {
|
||||
dest, err := net.DialTimeout("tcp", s.mapping.Destination, 10*time.Second)
|
||||
if err != nil {
|
||||
lg := s.conn()
|
||||
log.Printf("stream %s: dial %s failed: %v", lg, s.mapping.Destination, err)
|
||||
lg.wc.removeStream(lg.sid)
|
||||
lg.wc.sendRst(lg.sid)
|
||||
log.Printf("stream %s: dial %s failed: %v", s.name(), s.mapping.Destination, err)
|
||||
if wc := s.conn(); wc != nil {
|
||||
wc.detach()
|
||||
wc.sendRst()
|
||||
}
|
||||
s.teardown(false)
|
||||
return
|
||||
}
|
||||
@@ -616,7 +492,7 @@ func (s *Stream) run() {
|
||||
if s.mapping.ProxyProtocol {
|
||||
if hdr := s.buildProxyHeader(dest); hdr != nil {
|
||||
if _, err := dest.Write(hdr); err != nil {
|
||||
log.Printf("stream %s: proxy header write: %v", s.conn(), err)
|
||||
log.Printf("stream %s: proxy header write: %v", s.name(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -663,7 +539,7 @@ func (s *Stream) run() {
|
||||
}
|
||||
|
||||
// sendToHub forwards destination bytes to the hub in bounded DATA frames,
|
||||
// honoring both the stream send window and the client-wide bandwidth cap.
|
||||
// honoring both the send window and the client-wide bandwidth cap.
|
||||
// Returns false once the stream closed or the worker conn failed.
|
||||
func (s *Stream) sendToHub(data []byte) bool {
|
||||
for len(data) > 0 {
|
||||
@@ -674,7 +550,7 @@ func (s *Stream) sendToHub(data []byte) bool {
|
||||
// Credit first, bandwidth second. The reverse order would charge the
|
||||
// budget for bytes still parked on an exhausted window, so the client
|
||||
// would throttle itself below the configured rate. Holding credit while
|
||||
// waiting for tokens is free — credit is per-stream, and the hub returns
|
||||
// waiting for tokens is free — credit is per-tunnel, and the hub returns
|
||||
// it as it drains data to the player, independent of our pacing.
|
||||
if !s.acquireSendWnd(n) {
|
||||
return false
|
||||
@@ -714,9 +590,6 @@ func (s *Stream) sendToHub(data []byte) bool {
|
||||
// one taken before the attempt. That also makes a failure survivable: while the
|
||||
// stream can still be resumed the bytes are already safe, and the reattach
|
||||
// replays them from wherever the hub says it got to.
|
||||
//
|
||||
// Note the old code let a failed write drop the rest of the chunk on the floor —
|
||||
// the caller's slice advance sat after the error return.
|
||||
func (s *Stream) emit(chunk []byte) bool {
|
||||
s.sendMu.Lock()
|
||||
if s.resumable {
|
||||
@@ -725,8 +598,13 @@ func (s *Stream) emit(chunk []byte) bool {
|
||||
s.un.advance(s.ackedOffset.Load())
|
||||
s.un.append(chunk)
|
||||
}
|
||||
lg := s.conn()
|
||||
err := lg.wc.sendData(lg.sid, chunk)
|
||||
wc := s.conn()
|
||||
var err error
|
||||
if wc != nil {
|
||||
err = wc.sendData(chunk)
|
||||
} else {
|
||||
err = errPoolClosed
|
||||
}
|
||||
s.sendMu.Unlock()
|
||||
|
||||
if err == nil {
|
||||
@@ -792,7 +670,7 @@ func (s *Stream) writeLoop() {
|
||||
|
||||
// deliverFromHub enqueues hub bytes for the destination. Called from the worker
|
||||
// readLoop; it never blocks — a peer that exceeds the advertised window is a
|
||||
// protocol violator and gets the stream reset.
|
||||
// protocol violator and gets the tunnel reset.
|
||||
func (s *Stream) deliverFromHub(data []byte) {
|
||||
if s.vel != nil {
|
||||
s.vel.ObserveC2S(data) // observation only; bytes still forwarded verbatim
|
||||
@@ -804,10 +682,11 @@ func (s *Stream) deliverFromHub(data []byte) {
|
||||
}
|
||||
if s.qBytes+len(data) > s.client.streamWnd {
|
||||
s.mu.Unlock()
|
||||
lg := s.conn()
|
||||
log.Printf("stream %s: peer exceeded flow-control window; resetting", lg)
|
||||
lg.wc.removeStream(lg.sid)
|
||||
lg.wc.sendRst(lg.sid)
|
||||
log.Printf("stream %s: peer exceeded flow-control window; resetting", s.name())
|
||||
if wc := s.conn(); wc != nil {
|
||||
wc.detach()
|
||||
wc.sendRst()
|
||||
}
|
||||
s.teardown(false)
|
||||
return
|
||||
}
|
||||
@@ -891,8 +770,9 @@ func (s *Stream) credit(n int) {
|
||||
delta := s.consumed
|
||||
s.consumed = 0
|
||||
s.mu.Unlock()
|
||||
lg := s.conn()
|
||||
lg.wc.sendWndUpdate(lg.sid, delta)
|
||||
if wc := s.conn(); wc != nil {
|
||||
wc.sendWndUpdate(delta)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Stream) buildProxyHeader(dest net.Conn) []byte {
|
||||
@@ -957,10 +837,11 @@ func (s *Stream) teardown(notifyHub bool) {
|
||||
if dest != nil {
|
||||
_ = dest.Close()
|
||||
}
|
||||
lg := s.conn()
|
||||
lg.wc.removeStream(lg.sid)
|
||||
if notifyHub {
|
||||
lg.wc.sendFin(lg.sid)
|
||||
if wc := s.conn(); wc != nil {
|
||||
wc.detach()
|
||||
if notifyHub {
|
||||
wc.sendFin()
|
||||
}
|
||||
}
|
||||
s.logSummary()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user