redapricot/client/worker.go

package client

import (
	"log"
	"net"
	"sync"
	"time"

	"github.com/iceBear67/redapricot/client/wire"
)

// WorkerPool manages up to maxConn worker connections and allocates streams
// using the least-loaded strategy (PROTOCOL.md §7.1).
type WorkerPool struct {
	client  *Client
	maxConn int

	mu    sync.Mutex
	conns []*WorkerConn
}

func newWorkerPool(c *Client, maxConn int) *WorkerPool {
	return &WorkerPool{client: c, maxConn: maxConn}
}

// Allocate returns a worker conn and a fresh stream id to place a new stream on.
func (p *WorkerPool) Allocate() (*WorkerConn, int, error) {
	p.mu.Lock()
	defer p.mu.Unlock()

	var best *WorkerConn
	bestCount := 0
	for _, wc := range p.conns {
		n := wc.streamCount()
		if best == nil || n < bestCount {
			best = wc
			bestCount = n
		}
	}

	needNew := best == nil || (bestCount > SaturationThreshold && len(p.conns) < p.maxConn)
	if needNew {
		wc, err := p.dialWorker()
		if err != nil {
			if best == nil {
				return nil, 0, err
			}
			log.Printf("worker dial failed, reusing existing conn: %v", err)
		} else {
			p.conns = append(p.conns, wc)
			best = wc
		}
	}
	return best, best.newSid(), nil
}

func (p *WorkerPool) dialWorker() (*WorkerConn, error) {
	fc, peerWnd, err := p.client.dialSession(MagicWorker)
	if err != nil {
		return nil, err
	}
	wc := &WorkerConn{
		pool:        p,
		fc:          fc,
		sendWndInit: peerWnd,
		recvWndInit: p.client.streamWnd,
		streams:     make(map[int]*Stream),
		nextSid:     1,
	}
	go wc.readLoop()
	log.Printf("opened worker conn (#%d in pool, send window %d, recv window %d)",
		len(p.conns)+1, wc.sendWndInit, wc.recvWndInit)
	return wc, nil
}

func (p *WorkerPool) count() int {
	p.mu.Lock()
	defer p.mu.Unlock()
	return len(p.conns)
}

func (p *WorkerPool) remove(wc *WorkerConn) {
	p.mu.Lock()
	defer p.mu.Unlock()
	for i, c := range p.conns {
		if c == wc {
			p.conns = append(p.conns[:i], p.conns[i+1:]...)
			return
		}
	}
}

func (p *WorkerPool) closeAll() {
	p.mu.Lock()
	conns := append([]*WorkerConn(nil), p.conns...)
	p.mu.Unlock()
	for _, wc := range conns {
		_ = wc.fc.Close()
	}
}

// WorkerConn is one multiplexed worker connection to the hub.
type WorkerConn struct {
	pool *WorkerPool
	fc   *wire.FramedConn

	sendWndInit int // hub's advertised per-stream receive window (our send budget)
	recvWndInit int // our advertised per-stream receive window (bounds each recv queue)

	mu      sync.Mutex
	streams map[int]*Stream
	nextSid int
}

func (wc *WorkerConn) streamCount() int {
	wc.mu.Lock()
	defer wc.mu.Unlock()
	return len(wc.streams)
}

func (wc *WorkerConn) newSid() int {
	wc.mu.Lock()
	defer wc.mu.Unlock()
	sid := wc.nextSid
	wc.nextSid++
	return sid
}

func (wc *WorkerConn) registerStream(sid int, st *Stream) {
	wc.mu.Lock()
	wc.streams[sid] = st
	wc.mu.Unlock()
}

func (wc *WorkerConn) getStream(sid int) *Stream {
	wc.mu.Lock()
	defer wc.mu.Unlock()
	return wc.streams[sid]
}

func (wc *WorkerConn) removeStream(sid int) *Stream {
	wc.mu.Lock()
	defer wc.mu.Unlock()
	st := wc.streams[sid]
	delete(wc.streams, sid)
	return st
}

// readLoop dispatches inbound mux frames. It must never block on a stream's
// destination: DATA is only enqueued (the per-stream writeLoop does the actual
// destination writes), so one slow destination cannot stall other streams.
func (wc *WorkerConn) readLoop() {
	for {
		payload, err := wc.fc.ReadFrame()
		if err != nil {
			break
		}
		r := wire.NewReader(payload)
		ftype, err := r.U8()
		if err != nil {
			continue
		}
		sid, err := r.VarInt()
		if err != nil {
			continue
		}
		switch ftype {
		case MuxData:
			if st := wc.getStream(sid); st != nil {
				st.deliverFromHub(r.Remaining())
			}
		case MuxWnd:
			if delta, err := r.VarInt(); err == nil && delta > 0 {
				if st := wc.getStream(sid); st != nil {
					st.grantSendWnd(delta)
				}
			}
		case MuxFin:
			// Graceful: drain what is already queued to the destination first.
			if st := wc.removeStream(sid); st != nil {
				st.gracefulFin()
			}
		case MuxRst:
			if st := wc.removeStream(sid); st != nil {
				st.teardown(false)
			}
		default:
			log.Printf("worker: unknown mux type %d", ftype)
		}
	}
	// Connection lost: tear down all streams and drop from pool.
	wc.pool.remove(wc)
	wc.mu.Lock()
	streams := make([]*Stream, 0, len(wc.streams))
	for _, st := range wc.streams {
		streams = append(streams, st)
	}
	wc.streams = make(map[int]*Stream)
	wc.mu.Unlock()
	for _, st := range streams {
		st.teardown(false)
	}
}

func (wc *WorkerConn) sendSyn(sid int, cid []byte) {
	_ = wc.fc.WriteFrame(wire.NewWriter().U8(MuxSyn).VarInt(sid).Bytes(cid).Out())
}

func (wc *WorkerConn) sendData(sid int, data []byte) error {
	return wc.fc.WriteFrame(wire.NewWriter().U8(MuxData).VarInt(sid).Bytes(data).Out())
}

func (wc *WorkerConn) sendFin(sid int) {
	_ = wc.fc.WriteFrame(wire.NewWriter().U8(MuxFin).VarInt(sid).Out())
}

func (wc *WorkerConn) sendRst(sid int) {
	_ = wc.fc.WriteFrame(wire.NewWriter().U8(MuxRst).VarInt(sid).Out())
}

func (wc *WorkerConn) sendWndUpdate(sid, delta int) {
	_ = wc.fc.WriteFrame(wire.NewWriter().U8(MuxWnd).VarInt(sid).VarInt(delta).Out())
}

// Stream bridges one player (via the hub) to one destination connection.
//
// Data from the hub is queued and written to the destination by a dedicated
// writeLoop goroutine. The queue is bounded by the advertised receive window —
// the hub never sends more un-credited bytes, so overflow is a protocol
// violation and resets the stream.
type Stream struct {
	wc      *WorkerConn
	sid     int
	cid     []byte
	mapping Mapping
	srcIP   string
	srcPort int

	mu         sync.Mutex
	cond       *sync.Cond
	dest       net.Conn
	connected  bool
	closed     bool
	finPending bool     // hub sent FIN; close the destination once the queue drains
	q          [][]byte // hub -> destination, waiting for writeLoop
	qBytes     int
	sendWnd    int // flow control: budget for destination -> hub DATA
	consumed   int // flow control: drained bytes not yet credited back to the hub
}

func newStream(wc *WorkerConn, sid int, cid []byte, m Mapping, ip string, port int) *Stream {
	s := &Stream{wc: wc, sid: sid, cid: cid, mapping: m, srcIP: ip, srcPort: port, sendWnd: wc.sendWndInit}
	s.cond = sync.NewCond(&s.mu)
	return s
}

// run dials the destination, optionally writes the PROXY v2 header, then pumps
// destination -> hub (respecting the stream send window when negotiated).
func (s *Stream) run() {
	dest, err := net.DialTimeout("tcp", s.mapping.Destination, 10*time.Second)
	if err != nil {
		log.Printf("stream %d: dial %s failed: %v", s.sid, s.mapping.Destination, err)
		s.wc.removeStream(s.sid)
		s.wc.sendRst(s.sid)
		s.teardown(false)
		return
	}
	if tcp, ok := dest.(*net.TCPConn); ok {
		_ = tcp.SetNoDelay(true)
	}

	if s.mapping.ProxyProtocol {
		if hdr := s.buildProxyHeader(dest); hdr != nil {
			if _, err := dest.Write(hdr); err != nil {
				log.Printf("stream %d: proxy header write: %v", s.sid, err)
			}
		}
	}

	s.mu.Lock()
	if s.closed {
		s.mu.Unlock()
		_ = dest.Close()
		return
	}
	s.dest = dest
	s.connected = true
	s.cond.Broadcast() // wake writeLoop: queued hub bytes can flow now
	s.mu.Unlock()

	// destination -> hub
	buf := make([]byte, DataChunkSize)
	for {
		n, err := dest.Read(buf)
		if n > 0 {
			if !s.acquireSendWnd(n) {
				break
			}
			if werr := s.wc.sendData(s.sid, buf[:n]); werr != nil {
				break
			}
		}
		if err != nil {
			break
		}
	}
	s.teardown(true)
}

// writeLoop is the only writer to the destination. It drains the receive queue,
// credits the hub as bytes land on the destination socket, and performs the
// deferred graceful close when a FIN arrived with data still queued.
func (s *Stream) writeLoop() {
	for {
		s.mu.Lock()
		for !s.closed && (!s.connected || (len(s.q) == 0 && !s.finPending)) {
			s.cond.Wait()
		}
		if s.closed {
			s.mu.Unlock()
			return
		}
		if len(s.q) == 0 { // finPending and fully drained
			s.mu.Unlock()
			s.teardown(false)
			return
		}
		data := s.q[0]
		s.q = s.q[1:]
		s.qBytes -= len(data)
		dest := s.dest
		s.mu.Unlock()

		if _, err := dest.Write(data); err != nil {
			s.teardown(true)
			return
		}
		s.credit(len(data))
	}
}

// deliverFromHub enqueues hub bytes for the destination. Called from the worker
// readLoop; it never blocks — a peer that exceeds the advertised window is a
// protocol violator and gets the stream reset.
func (s *Stream) deliverFromHub(data []byte) {
	s.mu.Lock()
	if s.closed || s.finPending {
		s.mu.Unlock()
		return
	}
	if s.qBytes+len(data) > s.wc.recvWndInit {
		s.mu.Unlock()
		log.Printf("stream %d: peer exceeded flow-control window; resetting", s.sid)
		s.wc.removeStream(s.sid)
		s.wc.sendRst(s.sid)
		s.teardown(false)
		return
	}
	s.q = append(s.q, data)
	s.qBytes += len(data)
	s.cond.Broadcast()
	s.mu.Unlock()
}

// acquireSendWnd blocks until the stream may send n more bytes to the hub.
// Returns false if the stream closed while waiting.
func (s *Stream) acquireSendWnd(n int) bool {
	s.mu.Lock()
	defer s.mu.Unlock()
	for !s.closed && s.sendWnd < n {
		s.cond.Wait()
	}
	if s.closed {
		return false
	}
	s.sendWnd -= n
	return true
}

func (s *Stream) grantSendWnd(delta int) {
	s.mu.Lock()
	s.sendWnd += delta
	s.cond.Broadcast()
	s.mu.Unlock()
}

// credit accounts bytes drained to the destination and grants the hub more
// window once half of our receive window has been consumed.
func (s *Stream) credit(n int) {
	s.mu.Lock()
	s.consumed += n
	if s.closed || s.consumed*2 < s.wc.recvWndInit {
		s.mu.Unlock()
		return
	}
	delta := s.consumed
	s.consumed = 0
	s.mu.Unlock()
	s.wc.sendWndUpdate(s.sid, delta)
}

func (s *Stream) buildProxyHeader(dest net.Conn) []byte {
	srcIP := net.ParseIP(s.srcIP)
	if srcIP == nil {
		return nil
	}
	dstTCP, ok := dest.RemoteAddr().(*net.TCPAddr)
	if !ok {
		return nil
	}
	return BuildProxyV2(srcIP, s.srcPort, dstTCP.IP, dstTCP.Port)
}

// finDrainTimeout bounds how long a FIN'd stream may keep draining its queue
// into the destination, so a dead destination cannot hold the stream forever.
const finDrainTimeout = 10 * time.Second

// gracefulFin marks the hub-side close; writeLoop finishes the queue and then
// tears the stream down.
func (s *Stream) gracefulFin() {
	s.mu.Lock()
	if s.closed || s.finPending {
		s.mu.Unlock()
		return
	}
	s.finPending = true
	if s.dest != nil {
		_ = s.dest.SetWriteDeadline(time.Now().Add(finDrainTimeout))
	}
	s.cond.Broadcast()
	s.mu.Unlock()
}

// teardown closes the stream immediately; notifyHub sends a FIN when true.
// Idempotent; wakes every goroutine parked on the stream.
func (s *Stream) teardown(notifyHub bool) {
	s.mu.Lock()
	if s.closed {
		s.mu.Unlock()
		return
	}
	s.closed = true
	dest := s.dest
	s.cond.Broadcast()
	s.mu.Unlock()

	if dest != nil {
		_ = dest.Close()
	}
	s.wc.removeStream(s.sid)
	if notifyHub {
		s.wc.sendFin(s.sid)
	}
}