redapricot/client/resume.go

package client

import (
	"errors"
	"log"
	"time"

	"github.com/iceBear67/redapricot/client/wire"
)

// Stream resumption (PROTOCOL.md §7.5).
//
// A worker conn carries many players but is only the middle leg of each: when
// it dies, both terminal sockets are usually still perfectly healthy. Tearing
// the streams down therefore throws away working connections because a
// replaceable transport failed — one conntrack expiry disconnects everyone on
// that conn.
//
// Instead the stream parks: the destination socket stays open, the hub hangs the
// player socket, and the client reattaches over a fresh conn. Correctness rests
// on retransmission being byte-exact. Bytes handed to a dying socket are lost
// with no notification, and the frame cipher cannot be resynchronized, so each
// side replays from the offset the other reports it accepted.

var (
	errResumeUnknown  = errors.New("hub does not know this stream")
	errResumeRaced    = errors.New("hub has this stream bound to another conn")
	errResumeRefused  = errors.New("hub refused the reattach")
	errResumeTimeout  = errors.New("no RESUME_ACK from the hub")
	errResumeConnLost = errors.New("the conn carrying the reattach died")
	errResumeTooOld   = errors.New("hub accepted past what we still hold")
	errResumeNoResume = errors.New("hub does not support stream resumption")
)

// resumeGrace is how long a stream parked from a conn may keep trying, clamped
// under what the hub advertised. The client must always give up first: a hub
// that drops the player while we are still reattaching would leave us pumping a
// destination nobody is reading.
func (c *Client) resumeGrace(hubGrace time.Duration) time.Duration {
	grace := c.cfg.resumeGrace()
	if hubGrace > 0 && hubGrace < grace {
		grace = hubGrace
	}
	return grace
}

// park suspends a stream whose worker conn died instead of destroying it, and
// starts trying to reattach. Reports false when the stream cannot be parked, in
// which case the caller tears it down as before.
//
// A stream already closing (finPending) is not parked: the hub has said the
// player is gone, so there is nothing left to preserve.
func (s *Stream) park(grace time.Duration) bool {
	if !s.resumable {
		return false
	}
	s.mu.Lock()
	if s.closed || s.finPending {
		s.mu.Unlock()
		return false
	}
	already := s.parked
	s.parked = true
	if s.stats != nil && !already {
		s.parkedAt = time.Now()
	}
	s.mu.Unlock()

	if already {
		// A reattach was already in flight and had registered this stream on the
		// conn that just died — which is how it got here at all. That attempt
		// still owns the stream, so reporting failure would have the caller tear
		// down a player that is mid-recovery. Fail its wait immediately rather
		// than let it sit out the ack timeout: the grace budget is small, and
		// spending ten seconds of it waiting on a socket that is already gone is
		// the difference between reattaching and dropping the player.
		s.deliverResume(resumeResult{err: errResumeConnLost})
		return true
	}
	go s.resumeLoop(grace)
	return true
}

// resumeLoop reattaches the stream, retrying until it succeeds or the grace
// period runs out.
//
// Attempts are started right up to the deadline rather than reserving a whole
// dial's worth of budget for the last one. Reserving it would be self-defeating
// — the grace and HandshakeTimeout are the same order of magnitude, so the
// reservation can consume the entire budget and leave no attempt at all — and
// overshooting is safe: an attempt that lands after the hub has dropped the
// player is answered with RST(unknown stream) and tears down cleanly.
func (s *Stream) resumeLoop(grace time.Duration) {
	deadline := time.Now().Add(grace)
	for {
		if s.isClosed() {
			return
		}
		if !time.Now().Before(deadline) {
			break
		}
		err := s.tryResume()
		if err == nil {
			return
		}
		// errResumeRaced falls through to the retry below. The hub has this
		// stream bound to a conn that is not ours — a half-open conn whose
		// death the hub has not yet learned, or a bind left behind by a racing
		// attempt on a now-dead conn. There is no other live attempt: park is
		// the only resumeLoop starter and it refuses to double-start. Retrying
		// is safe precisely because nothing else owns the stream — the foreign
		// bind dies with its conn, the hub re-parks, and a later RESUME lands.
		// The grace deadline bounds the loop and expiry tears the stream down,
		// so a hub that never re-parks cannot hang us forever.
		if errors.Is(err, errResumeUnknown) || errors.Is(err, errResumeTooOld) {
			// Terminal: the hub has no state for this stream (it restarted, the
			// grace expired, or a load balancer sent us to a different instance).
			log.Printf("stream resume abandoned: %v", err)
			s.teardown(false)
			return
		}
		select {
		case <-s.done:
			return
		case <-time.After(ResumeRetryDelay):
		}
	}
	log.Printf("stream resume gave up after %s; closing destination", grace)
	// No FIN: the only conns we could send it on are the ones that just failed
	// us. The hub drops the hanging player when its own grace expires.
	s.teardown(false)
}

// tryResume performs one reattach attempt: find a live conn, claim a stream id
// on it, send RESUME, and replay from wherever the hub says it got to.
func (s *Stream) tryResume() error {
	wc, sid, err := s.allocateForResume()
	if err != nil {
		return err
	}

	s.mu.Lock()
	cid := s.cid
	accepted := s.acceptedOffset
	delivered := s.deliveredOffset
	wait := make(chan resumeResult, 1)
	s.resumeWait = wait
	s.mu.Unlock()

	msg := wire.NewWriter().U8(MuxResume).VarInt(sid).Bytes(cid).
		I64(accepted).I64(delivered).Out()
	if err := wc.fc.WriteFrame(msg); err != nil {
		s.abandonAttempt(wc, sid)
		return err
	}

	var res resumeResult
	select {
	case res = <-wait:
	case <-s.done:
		// Torn down while waiting. teardown only deregisters the leg the stream
		// was bound to, which is not this one, so the claim made above has to be
		// withdrawn here or it stays in the new conn's table forever.
		s.abandonAttempt(wc, sid)
		return errResumeRefused
	case <-time.After(ResumeAckTimeout):
		s.abandonAttempt(wc, sid)
		return errResumeTimeout
	}
	if res.err != nil {
		s.abandonAttempt(wc, sid)
		return res.err
	}
	return s.completeResume(wc, sid, res)
}

// allocateForResume picks a live conn that will honour a reattach.
func (s *Stream) allocateForResume() (*WorkerConn, int, error) {
	for attempt := 0; attempt < allocateAttempts; attempt++ {
		wc, sid, err := s.client.pool.Allocate()
		if err != nil {
			return nil, 0, err
		}
		// Re-checked per conn, not assumed from the dead one: this may be a
		// different or restarted hub. Sending RESUME to a hub that does not know
		// the frame type would hang the player for the rest of the grace waiting
		// for an answer that is never coming.
		if !wc.resume {
			return nil, 0, errResumeNoResume
		}
		if wc.registerStream(sid, s) {
			return wc, sid, nil
		}
	}
	return nil, 0, errResumeRefused
}

// abandonAttempt withdraws a failed attempt from the conn it was made on, so a
// retry can never leave two RESUMEs outstanding for one stream.
func (s *Stream) abandonAttempt(wc *WorkerConn, sid int) {
	wc.removeStream(sid)
	s.mu.Lock()
	s.resumeWait = nil
	s.mu.Unlock()
}

// completeResume rebinds the stream to its new conn and replays what the hub is
// missing, holding sendMu throughout so live traffic cannot overtake the replay.
func (s *Stream) completeResume(wc *WorkerConn, sid int, res resumeResult) error {
	s.sendMu.Lock()
	// Delivery is a strictly stronger fact than credit — the hub only credits what
	// it has delivered — so the reported offset can be adopted wholesale. Doing so
	// also repairs the ledger: the grants destroyed by the outage are exactly the
	// gap between the two, and without this the retained region would carry that
	// dead prefix for the rest of the stream's life.
	s.ackedOffset.Store(res.delivered)
	s.un.advance(res.delivered)
	replay := s.un.from(res.accepted)
	if replay == nil {
		s.sendMu.Unlock()
		s.abandonAttempt(wc, sid)
		return errResumeTooOld
	}
	// Three offsets, three jobs, and conflating any two of them breaks something
	// different.
	//
	// What to replay is measured from what the hub *accepted* — the bytes it
	// never received. What the window should be is measured from what it
	// *delivered*, because the window is a promise about undelivered bytes.
	// It cannot be measured from what it *credited*: credit arrives as deltas,
	// and the grants in flight when the connection died are gone for good, so a
	// window derived from them would be permanently short — and, when a full
	// window was outstanding at the drop, permanently zero. That is a deadlock,
	// not a slowdown: no credit can arrive because nothing can be sent.
	outstanding := s.un.length()
	replayed := s.un.end() - res.accepted

	// Publish the new binding before any frame goes out on it, and as one value:
	// stream ids restart at 1 per conn, so a half-updated pair would address a
	// different player's stream.
	s.leg.Store(&leg{wc: wc, sid: sid})

	s.mu.Lock()
	// Restated, not patched. The window is a delta ledger and the outage tore a
	// hole in it; deriving it afresh from the delivered offset closes the hole
	// exactly, whatever was lost.
	s.sendWnd = wc.sendWndInit - int(outstanding)
	if s.sendWnd < 0 {
		s.sendWnd = 0
	}
	// Symmetrically, our own pending credit is discarded rather than flushed:
	// the delivered offset we reported already tells the hub everything those
	// deltas would have, and sending both would grant the same bytes twice.
	// Counting resumes from this baseline.
	s.consumed = 0
	if len(res.cid) == CIDLen {
		s.cid = res.cid // fresh capability, so a CID is never reusable twice
	}
	s.parked = false
	s.resumeWait = nil
	owedFin := s.finToHub
	s.cond.Broadcast() // release acquireSendWnd and any parked writer
	s.mu.Unlock()

	for len(replay) > 0 {
		n := len(replay)
		if n > s.client.chunk {
			n = s.client.chunk
		}
		if err := wc.sendData(sid, replay[:n]); err != nil {
			// The conn died mid-replay. The stream is still resumable, but not
			// from this leg — put it back in the parked state before returning
			// so the conn's teardown takes park()'s already-branch instead of
			// starting a second resumeLoop. The loop we came from keeps
			// retrying with the fresh CID, which the hub re-parked alongside
			// the stream when this conn died. Without this re-park the flag
			// cleared above would let two loops race one stream, and a racing
			// RST would then strand it with neither loop alive.
			s.mu.Lock()
			s.parked = true
			s.mu.Unlock()
			s.sendMu.Unlock()
			return err
		}
		replay = replay[n:]
	}
	s.sendMu.Unlock()

	// Counted only once the replay actually landed: a failed reattach above
	// returns before this, so a conn dying mid-replay does not inflate the
	// resume count with an attempt that never completed.
	if s.stats != nil {
		s.mu.Lock()
		s.stats.resumes++
		s.stats.hung += time.Since(s.parkedAt)
		s.stats.replayBytes += replayed
		s.mu.Unlock()
	}

	// A destination that closed while we were parked owed the hub a FIN that had
	// nowhere to go at the time.
	if owedFin {
		wc.sendFin(sid)
		s.teardown(false)
		return nil
	}
	log.Printf("stream conn%d/sid%d resumed (%d bytes replayed, %d outstanding)", wc.id, sid, replayed, outstanding)
	return nil
}

// deliverResume hands an answer to a reattach that is waiting for one. Reports
// false when no attempt was in flight, so the caller can treat the frame as it
// would on any live stream.
func (s *Stream) deliverResume(res resumeResult) bool {
	s.mu.Lock()
	ch := s.resumeWait
	s.resumeWait = nil
	s.mu.Unlock()
	if ch == nil {
		return false
	}
	ch <- res // buffered, and read at most once per attempt
	return true
}

// onRst applies an RST, using the reason to tell a stream that is genuinely gone
// from one that a racing reattach has taken over.
func (s *Stream) onRst(reason int) {
	err := errResumeRefused
	switch reason {
	case RstUnknownStream:
		err = errResumeUnknown
	case RstAlreadyBound:
		err = errResumeRaced
	}
	if s.deliverResume(resumeResult{err: err}) {
		return
	}
	s.teardown(false)
}

// noteFinWhileParked records a FIN the stream owes the hub but cannot send,
// because the only conn it has is the one that just died. Reports false when the
// stream is not parked and the caller should send it normally.
func (s *Stream) noteFinWhileParked() bool {
	s.mu.Lock()
	defer s.mu.Unlock()
	if !s.parked {
		return false
	}
	s.finToHub = true
	return true
}