package client import ( "errors" "log" "time" "github.com/iceBear67/redapricot/client/wire" ) // Stream resumption (PROTOCOL.md §7.5). // // A worker conn carries many players but is only the middle leg of each: when // it dies, both terminal sockets are usually still perfectly healthy. Tearing // the streams down therefore throws away working connections because a // replaceable transport failed — one conntrack expiry disconnects everyone on // that conn. // // Instead the stream parks: the destination socket stays open, the hub hangs the // player socket, and the client reattaches over a fresh conn. Correctness rests // on retransmission being byte-exact. Bytes handed to a dying socket are lost // with no notification, and the frame cipher cannot be resynchronized, so each // side replays from the offset the other reports it accepted. var ( errResumeUnknown = errors.New("hub does not know this stream") errResumeRaced = errors.New("hub has this stream bound to another conn") errResumeRefused = errors.New("hub refused the reattach") errResumeTimeout = errors.New("no RESUME_ACK from the hub") errResumeConnLost = errors.New("the conn carrying the reattach died") errResumeTooOld = errors.New("hub accepted past what we still hold") errResumeNoResume = errors.New("hub does not support stream resumption") ) // resumeGrace is how long a stream parked from a conn may keep trying, clamped // under what the hub advertised. The client must always give up first: a hub // that drops the player while we are still reattaching would leave us pumping a // destination nobody is reading. func (c *Client) resumeGrace(hubGrace time.Duration) time.Duration { grace := c.cfg.resumeGrace() if hubGrace > 0 && hubGrace < grace { grace = hubGrace } return grace } // park suspends a stream whose worker conn died instead of destroying it, and // starts trying to reattach. Reports false when the stream cannot be parked, in // which case the caller tears it down as before. // // A stream already closing (finPending) is not parked: the hub has said the // player is gone, so there is nothing left to preserve. func (s *Stream) park(grace time.Duration) bool { if !s.resumable { return false } s.mu.Lock() if s.closed || s.finPending { s.mu.Unlock() return false } already := s.parked s.parked = true if s.stats != nil && !already { s.parkedAt = time.Now() } s.mu.Unlock() if already { // A reattach was already in flight and had registered this stream on the // conn that just died — which is how it got here at all. That attempt // still owns the stream, so reporting failure would have the caller tear // down a player that is mid-recovery. Fail its wait immediately rather // than let it sit out the ack timeout: the grace budget is small, and // spending ten seconds of it waiting on a socket that is already gone is // the difference between reattaching and dropping the player. s.deliverResume(resumeResult{err: errResumeConnLost}) return true } go s.resumeLoop(grace) return true } // resumeLoop reattaches the stream, retrying until it succeeds or the grace // period runs out. // // Attempts are started right up to the deadline rather than reserving a whole // dial's worth of budget for the last one. Reserving it would be self-defeating // — the grace and HandshakeTimeout are the same order of magnitude, so the // reservation can consume the entire budget and leave no attempt at all — and // overshooting is safe: an attempt that lands after the hub has dropped the // player is answered with RST(unknown stream) and tears down cleanly. func (s *Stream) resumeLoop(grace time.Duration) { deadline := time.Now().Add(grace) for { if s.isClosed() { return } if !time.Now().Before(deadline) { break } err := s.tryResume() if err == nil { return } // errResumeRaced falls through to the retry below. The hub has this // stream bound to a conn that is not ours — a half-open conn whose // death the hub has not yet learned, or a bind left behind by a racing // attempt on a now-dead conn. There is no other live attempt: park is // the only resumeLoop starter and it refuses to double-start. Retrying // is safe precisely because nothing else owns the stream — the foreign // bind dies with its conn, the hub re-parks, and a later RESUME lands. // The grace deadline bounds the loop and expiry tears the stream down, // so a hub that never re-parks cannot hang us forever. if errors.Is(err, errResumeUnknown) || errors.Is(err, errResumeTooOld) { // Terminal: the hub has no state for this stream (it restarted, the // grace expired, or a load balancer sent us to a different instance). log.Printf("stream resume abandoned: %v", err) s.teardown(false) return } select { case <-s.done: return case <-time.After(ResumeRetryDelay): } } log.Printf("stream resume gave up after %s; closing destination", grace) // No FIN: the only conns we could send it on are the ones that just failed // us. The hub drops the hanging player when its own grace expires. s.teardown(false) } // tryResume performs one reattach attempt: find a live conn, claim a stream id // on it, send RESUME, and replay from wherever the hub says it got to. func (s *Stream) tryResume() error { wc, sid, err := s.allocateForResume() if err != nil { return err } s.mu.Lock() cid := s.cid accepted := s.acceptedOffset delivered := s.deliveredOffset wait := make(chan resumeResult, 1) s.resumeWait = wait s.mu.Unlock() msg := wire.NewWriter().U8(MuxResume).VarInt(sid).Bytes(cid). I64(accepted).I64(delivered).Out() if err := wc.fc.WriteFrame(msg); err != nil { s.abandonAttempt(wc, sid) return err } var res resumeResult select { case res = <-wait: case <-s.done: // Torn down while waiting. teardown only deregisters the leg the stream // was bound to, which is not this one, so the claim made above has to be // withdrawn here or it stays in the new conn's table forever. s.abandonAttempt(wc, sid) return errResumeRefused case <-time.After(ResumeAckTimeout): s.abandonAttempt(wc, sid) return errResumeTimeout } if res.err != nil { s.abandonAttempt(wc, sid) return res.err } return s.completeResume(wc, sid, res) } // allocateForResume picks a live conn that will honour a reattach. func (s *Stream) allocateForResume() (*WorkerConn, int, error) { for attempt := 0; attempt < allocateAttempts; attempt++ { wc, sid, err := s.client.pool.Allocate() if err != nil { return nil, 0, err } // Re-checked per conn, not assumed from the dead one: this may be a // different or restarted hub. Sending RESUME to a hub that does not know // the frame type would hang the player for the rest of the grace waiting // for an answer that is never coming. if !wc.resume { return nil, 0, errResumeNoResume } if wc.registerStream(sid, s) { return wc, sid, nil } } return nil, 0, errResumeRefused } // abandonAttempt withdraws a failed attempt from the conn it was made on, so a // retry can never leave two RESUMEs outstanding for one stream. func (s *Stream) abandonAttempt(wc *WorkerConn, sid int) { wc.removeStream(sid) s.mu.Lock() s.resumeWait = nil s.mu.Unlock() } // completeResume rebinds the stream to its new conn and replays what the hub is // missing, holding sendMu throughout so live traffic cannot overtake the replay. func (s *Stream) completeResume(wc *WorkerConn, sid int, res resumeResult) error { s.sendMu.Lock() // Delivery is a strictly stronger fact than credit — the hub only credits what // it has delivered — so the reported offset can be adopted wholesale. Doing so // also repairs the ledger: the grants destroyed by the outage are exactly the // gap between the two, and without this the retained region would carry that // dead prefix for the rest of the stream's life. s.ackedOffset.Store(res.delivered) s.un.advance(res.delivered) replay := s.un.from(res.accepted) if replay == nil { s.sendMu.Unlock() s.abandonAttempt(wc, sid) return errResumeTooOld } // Three offsets, three jobs, and conflating any two of them breaks something // different. // // What to replay is measured from what the hub *accepted* — the bytes it // never received. What the window should be is measured from what it // *delivered*, because the window is a promise about undelivered bytes. // It cannot be measured from what it *credited*: credit arrives as deltas, // and the grants in flight when the connection died are gone for good, so a // window derived from them would be permanently short — and, when a full // window was outstanding at the drop, permanently zero. That is a deadlock, // not a slowdown: no credit can arrive because nothing can be sent. outstanding := s.un.length() replayed := s.un.end() - res.accepted // Publish the new binding before any frame goes out on it, and as one value: // stream ids restart at 1 per conn, so a half-updated pair would address a // different player's stream. s.leg.Store(&leg{wc: wc, sid: sid}) s.mu.Lock() // Restated, not patched. The window is a delta ledger and the outage tore a // hole in it; deriving it afresh from the delivered offset closes the hole // exactly, whatever was lost. s.sendWnd = wc.sendWndInit - int(outstanding) if s.sendWnd < 0 { s.sendWnd = 0 } // Symmetrically, our own pending credit is discarded rather than flushed: // the delivered offset we reported already tells the hub everything those // deltas would have, and sending both would grant the same bytes twice. // Counting resumes from this baseline. s.consumed = 0 if len(res.cid) == CIDLen { s.cid = res.cid // fresh capability, so a CID is never reusable twice } s.parked = false s.resumeWait = nil owedFin := s.finToHub s.cond.Broadcast() // release acquireSendWnd and any parked writer s.mu.Unlock() for len(replay) > 0 { n := len(replay) if n > s.client.chunk { n = s.client.chunk } if err := wc.sendData(sid, replay[:n]); err != nil { // The conn died mid-replay. The stream is still resumable, but not // from this leg — put it back in the parked state before returning // so the conn's teardown takes park()'s already-branch instead of // starting a second resumeLoop. The loop we came from keeps // retrying with the fresh CID, which the hub re-parked alongside // the stream when this conn died. Without this re-park the flag // cleared above would let two loops race one stream, and a racing // RST would then strand it with neither loop alive. s.mu.Lock() s.parked = true s.mu.Unlock() s.sendMu.Unlock() return err } replay = replay[n:] } s.sendMu.Unlock() // Counted only once the replay actually landed: a failed reattach above // returns before this, so a conn dying mid-replay does not inflate the // resume count with an attempt that never completed. if s.stats != nil { s.mu.Lock() s.stats.resumes++ s.stats.hung += time.Since(s.parkedAt) s.stats.replayBytes += replayed s.mu.Unlock() } // A destination that closed while we were parked owed the hub a FIN that had // nowhere to go at the time. if owedFin { wc.sendFin(sid) s.teardown(false) return nil } log.Printf("stream conn%d/sid%d resumed (%d bytes replayed, %d outstanding)", wc.id, sid, replayed, outstanding) return nil } // deliverResume hands an answer to a reattach that is waiting for one. Reports // false when no attempt was in flight, so the caller can treat the frame as it // would on any live stream. func (s *Stream) deliverResume(res resumeResult) bool { s.mu.Lock() ch := s.resumeWait s.resumeWait = nil s.mu.Unlock() if ch == nil { return false } ch <- res // buffered, and read at most once per attempt return true } // onRst applies an RST, using the reason to tell a stream that is genuinely gone // from one that a racing reattach has taken over. func (s *Stream) onRst(reason int) { err := errResumeRefused switch reason { case RstUnknownStream: err = errResumeUnknown case RstAlreadyBound: err = errResumeRaced } if s.deliverResume(resumeResult{err: err}) { return } s.teardown(false) } // noteFinWhileParked records a FIN the stream owes the hub but cannot send, // because the only conn it has is the one that just died. Reports false when the // stream is not parked and the caller should send it normally. func (s *Stream) noteFinWhileParked() bool { s.mu.Lock() defer s.mu.Unlock() if !s.parked { return false } s.finToHub = true return true }