package client import ( "log" "net" "sync" "time" "github.com/iceBear67/redapricot/client/wire" ) // WorkerPool manages up to maxConn worker connections and allocates streams // using the least-loaded strategy (PROTOCOL.md §7.1). type WorkerPool struct { client *Client maxConn int mu sync.Mutex conns []*WorkerConn } func newWorkerPool(c *Client, maxConn int) *WorkerPool { return &WorkerPool{client: c, maxConn: maxConn} } // Allocate returns a worker conn and a fresh stream id to place a new stream on. func (p *WorkerPool) Allocate() (*WorkerConn, int, error) { p.mu.Lock() defer p.mu.Unlock() var best *WorkerConn bestCount := 0 for _, wc := range p.conns { n := wc.streamCount() if best == nil || n < bestCount { best = wc bestCount = n } } needNew := best == nil || (bestCount > SaturationThreshold && len(p.conns) < p.maxConn) if needNew { wc, err := p.dialWorker() if err != nil { if best == nil { return nil, 0, err } log.Printf("worker dial failed, reusing existing conn: %v", err) } else { p.conns = append(p.conns, wc) best = wc } } return best, best.newSid(), nil } func (p *WorkerPool) dialWorker() (*WorkerConn, error) { fc, peerWnd, err := p.client.dialSession(MagicWorker) if err != nil { return nil, err } wc := &WorkerConn{ pool: p, fc: fc, sendWndInit: peerWnd, recvWndInit: p.client.streamWnd, streams: make(map[int]*Stream), nextSid: 1, } go wc.readLoop() log.Printf("opened worker conn (#%d in pool, send window %d, recv window %d)", len(p.conns)+1, wc.sendWndInit, wc.recvWndInit) return wc, nil } func (p *WorkerPool) count() int { p.mu.Lock() defer p.mu.Unlock() return len(p.conns) } func (p *WorkerPool) remove(wc *WorkerConn) { p.mu.Lock() defer p.mu.Unlock() for i, c := range p.conns { if c == wc { p.conns = append(p.conns[:i], p.conns[i+1:]...) return } } } func (p *WorkerPool) closeAll() { p.mu.Lock() conns := append([]*WorkerConn(nil), p.conns...) p.mu.Unlock() for _, wc := range conns { _ = wc.fc.Close() } } // WorkerConn is one multiplexed worker connection to the hub. type WorkerConn struct { pool *WorkerPool fc *wire.FramedConn sendWndInit int // hub's advertised per-stream receive window (our send budget) recvWndInit int // our advertised per-stream receive window (bounds each recv queue) mu sync.Mutex streams map[int]*Stream nextSid int } func (wc *WorkerConn) streamCount() int { wc.mu.Lock() defer wc.mu.Unlock() return len(wc.streams) } func (wc *WorkerConn) newSid() int { wc.mu.Lock() defer wc.mu.Unlock() sid := wc.nextSid wc.nextSid++ return sid } func (wc *WorkerConn) registerStream(sid int, st *Stream) { wc.mu.Lock() wc.streams[sid] = st wc.mu.Unlock() } func (wc *WorkerConn) getStream(sid int) *Stream { wc.mu.Lock() defer wc.mu.Unlock() return wc.streams[sid] } func (wc *WorkerConn) removeStream(sid int) *Stream { wc.mu.Lock() defer wc.mu.Unlock() st := wc.streams[sid] delete(wc.streams, sid) return st } // readLoop dispatches inbound mux frames. It must never block on a stream's // destination: DATA is only enqueued (the per-stream writeLoop does the actual // destination writes), so one slow destination cannot stall other streams. func (wc *WorkerConn) readLoop() { for { payload, err := wc.fc.ReadFrame() if err != nil { break } r := wire.NewReader(payload) ftype, err := r.U8() if err != nil { continue } sid, err := r.VarInt() if err != nil { continue } switch ftype { case MuxData: if st := wc.getStream(sid); st != nil { st.deliverFromHub(r.Remaining()) } case MuxWnd: if delta, err := r.VarInt(); err == nil && delta > 0 { if st := wc.getStream(sid); st != nil { st.grantSendWnd(delta) } } case MuxFin: // Graceful: drain what is already queued to the destination first. if st := wc.removeStream(sid); st != nil { st.gracefulFin() } case MuxRst: if st := wc.removeStream(sid); st != nil { st.teardown(false) } default: log.Printf("worker: unknown mux type %d", ftype) } } // Connection lost: tear down all streams and drop from pool. wc.pool.remove(wc) wc.mu.Lock() streams := make([]*Stream, 0, len(wc.streams)) for _, st := range wc.streams { streams = append(streams, st) } wc.streams = make(map[int]*Stream) wc.mu.Unlock() for _, st := range streams { st.teardown(false) } } func (wc *WorkerConn) sendSyn(sid int, cid []byte) { _ = wc.fc.WriteFrame(wire.NewWriter().U8(MuxSyn).VarInt(sid).Bytes(cid).Out()) } func (wc *WorkerConn) sendData(sid int, data []byte) error { return wc.fc.WriteFrame(wire.NewWriter().U8(MuxData).VarInt(sid).Bytes(data).Out()) } func (wc *WorkerConn) sendFin(sid int) { _ = wc.fc.WriteFrame(wire.NewWriter().U8(MuxFin).VarInt(sid).Out()) } func (wc *WorkerConn) sendRst(sid int) { _ = wc.fc.WriteFrame(wire.NewWriter().U8(MuxRst).VarInt(sid).Out()) } func (wc *WorkerConn) sendWndUpdate(sid, delta int) { _ = wc.fc.WriteFrame(wire.NewWriter().U8(MuxWnd).VarInt(sid).VarInt(delta).Out()) } // Stream bridges one player (via the hub) to one destination connection. // // Data from the hub is queued and written to the destination by a dedicated // writeLoop goroutine. The queue is bounded by the advertised receive window — // the hub never sends more un-credited bytes, so overflow is a protocol // violation and resets the stream. type Stream struct { wc *WorkerConn sid int cid []byte mapping Mapping srcIP string srcPort int mu sync.Mutex cond *sync.Cond dest net.Conn connected bool closed bool finPending bool // hub sent FIN; close the destination once the queue drains q [][]byte // hub -> destination, waiting for writeLoop qBytes int sendWnd int // flow control: budget for destination -> hub DATA consumed int // flow control: drained bytes not yet credited back to the hub } func newStream(wc *WorkerConn, sid int, cid []byte, m Mapping, ip string, port int) *Stream { s := &Stream{wc: wc, sid: sid, cid: cid, mapping: m, srcIP: ip, srcPort: port, sendWnd: wc.sendWndInit} s.cond = sync.NewCond(&s.mu) return s } // run dials the destination, optionally writes the PROXY v2 header, then pumps // destination -> hub (respecting the stream send window when negotiated). func (s *Stream) run() { dest, err := net.DialTimeout("tcp", s.mapping.Destination, 10*time.Second) if err != nil { log.Printf("stream %d: dial %s failed: %v", s.sid, s.mapping.Destination, err) s.wc.removeStream(s.sid) s.wc.sendRst(s.sid) s.teardown(false) return } if tcp, ok := dest.(*net.TCPConn); ok { _ = tcp.SetNoDelay(true) } if s.mapping.ProxyProtocol { if hdr := s.buildProxyHeader(dest); hdr != nil { if _, err := dest.Write(hdr); err != nil { log.Printf("stream %d: proxy header write: %v", s.sid, err) } } } s.mu.Lock() if s.closed { s.mu.Unlock() _ = dest.Close() return } s.dest = dest s.connected = true s.cond.Broadcast() // wake writeLoop: queued hub bytes can flow now s.mu.Unlock() // destination -> hub buf := make([]byte, DataChunkSize) for { n, err := dest.Read(buf) if n > 0 { if !s.acquireSendWnd(n) { break } if werr := s.wc.sendData(s.sid, buf[:n]); werr != nil { break } } if err != nil { break } } s.teardown(true) } // writeLoop is the only writer to the destination. It drains the receive queue, // credits the hub as bytes land on the destination socket, and performs the // deferred graceful close when a FIN arrived with data still queued. func (s *Stream) writeLoop() { for { s.mu.Lock() for !s.closed && (!s.connected || (len(s.q) == 0 && !s.finPending)) { s.cond.Wait() } if s.closed { s.mu.Unlock() return } if len(s.q) == 0 { // finPending and fully drained s.mu.Unlock() s.teardown(false) return } data := s.q[0] s.q = s.q[1:] s.qBytes -= len(data) dest := s.dest s.mu.Unlock() if _, err := dest.Write(data); err != nil { s.teardown(true) return } s.credit(len(data)) } } // deliverFromHub enqueues hub bytes for the destination. Called from the worker // readLoop; it never blocks — a peer that exceeds the advertised window is a // protocol violator and gets the stream reset. func (s *Stream) deliverFromHub(data []byte) { s.mu.Lock() if s.closed || s.finPending { s.mu.Unlock() return } if s.qBytes+len(data) > s.wc.recvWndInit { s.mu.Unlock() log.Printf("stream %d: peer exceeded flow-control window; resetting", s.sid) s.wc.removeStream(s.sid) s.wc.sendRst(s.sid) s.teardown(false) return } s.q = append(s.q, data) s.qBytes += len(data) s.cond.Broadcast() s.mu.Unlock() } // acquireSendWnd blocks until the stream may send n more bytes to the hub. // Returns false if the stream closed while waiting. func (s *Stream) acquireSendWnd(n int) bool { s.mu.Lock() defer s.mu.Unlock() for !s.closed && s.sendWnd < n { s.cond.Wait() } if s.closed { return false } s.sendWnd -= n return true } func (s *Stream) grantSendWnd(delta int) { s.mu.Lock() s.sendWnd += delta s.cond.Broadcast() s.mu.Unlock() } // credit accounts bytes drained to the destination and grants the hub more // window once half of our receive window has been consumed. func (s *Stream) credit(n int) { s.mu.Lock() s.consumed += n if s.closed || s.consumed*2 < s.wc.recvWndInit { s.mu.Unlock() return } delta := s.consumed s.consumed = 0 s.mu.Unlock() s.wc.sendWndUpdate(s.sid, delta) } func (s *Stream) buildProxyHeader(dest net.Conn) []byte { srcIP := net.ParseIP(s.srcIP) if srcIP == nil { return nil } dstTCP, ok := dest.RemoteAddr().(*net.TCPAddr) if !ok { return nil } return BuildProxyV2(srcIP, s.srcPort, dstTCP.IP, dstTCP.Port) } // finDrainTimeout bounds how long a FIN'd stream may keep draining its queue // into the destination, so a dead destination cannot hold the stream forever. const finDrainTimeout = 10 * time.Second // gracefulFin marks the hub-side close; writeLoop finishes the queue and then // tears the stream down. func (s *Stream) gracefulFin() { s.mu.Lock() if s.closed || s.finPending { s.mu.Unlock() return } s.finPending = true if s.dest != nil { _ = s.dest.SetWriteDeadline(time.Now().Add(finDrainTimeout)) } s.cond.Broadcast() s.mu.Unlock() } // teardown closes the stream immediately; notifyHub sends a FIN when true. // Idempotent; wakes every goroutine parked on the stream. func (s *Stream) teardown(notifyHub bool) { s.mu.Lock() if s.closed { s.mu.Unlock() return } s.closed = true dest := s.dest s.cond.Broadcast() s.mu.Unlock() if dest != nil { _ = dest.Close() } s.wc.removeStream(s.sid) if notifyHub { s.wc.sendFin(s.sid) } }