// The userspace TCP/IP stack: lwIP lifecycle, one netif per live tunnel. // // --------------------------------------------------------------------------- // Threading // --------------------------------------------------------------------------- // lwIP built with NO_SYS=1 has no locking whatsoever. Every call into it and // every callback out of it must be serialized, so all of it runs on one asio // strand -- Stack::strand(). Handlers the caller supplies run on the executor // the caller passed in, never on the strand, so nothing above this module ever // touches lwIP state. // // A strand rather than a dedicated thread: lwIP requires "no concurrency", not // "the same thread", and a strand gives that plus the happens-before edges, // without adding a thread or a second io_context to reason about. // // --------------------------------------------------------------------------- // Why there can only be one Stack // --------------------------------------------------------------------------- // lwIP's state is file-scope globals: the netif list, tcp_active_pcbs, the // timeout wheel, the memp pools. There is exactly one lwIP per process and no // amount of wrapping changes that. The constructor enforces it rather than // letting a second instance silently corrupt the first. // // This matters because make-before-break (docs/ARCHITECTURE.md 5) needs two // tunnels alive at once. That works: one Stack, two Netifs, and every PCB // pinned to its netif with tcp_bind_netif()/udp_bind_netif() so routing cannot // send an old session's packets out the new tunnel. // // It works with one exception, and it is worth stating plainly. Attribution of // *inbound* packets is by destination address, so two tunnels that push the // same address are indistinguishable to lwIP. VPNGate servers hand out private // addresses from a small set of ranges, so this is not hypothetical. Netif // creation therefore rejects an address already in use, and the switch // controller treats that rejection as "graceful drain is impossible here" and // falls back to a hard switch -- which is the behaviour the requirements ask // for when connections cannot be preserved. // // --------------------------------------------------------------------------- // IPv6 // --------------------------------------------------------------------------- // Not supported. lwIP is compiled IPv4-only (see lwip_port/lwipopts.h): VPNGate // nodes essentially never push a usable IPv6 prefix, and carrying a second // address family through the netif, the resolver and the SOCKS5 layer would // roughly double the surface for something no node exercises. An IPv6 target // arriving over SOCKS5 is refused with "address type not supported" rather than // being quietly resolved to something else. #pragma once #include #include #include #include #include #include #include #include "common/config.h" #include "common/endpoint.h" #include "common/strand_deleter.h" #include "netstack/packet_link.h" #include "netstack/stream.h" struct pbuf; struct netif; namespace ovg::netstack { using ovg::Strand; class Netif; class LwipTcpStream; class LwipUdpSocket; class DnsResolver; // Implemented by everything a Netif can own. The netif holds these weakly, so // dropping the caller's reference still tears the connection down; this is only // the handle it needs to reach survivors when the tunnel goes away underneath // them. Called on the stack's strand. class Closable { public: virtual ~Closable() = default; virtual void abort_from_netif(const std::error_code &reason) = 0; }; // Everything a netif needs, extracted from what the VPN server pushed. struct NetifConfig { IpAddress address; // our address inside the tunnel int prefix = 0; // 1..32 IpAddress gateway; // may be unset on a point-to-point link int mtu = 1500; std::vector dns; // in server priority order std::string label; // node id; appears in every log line }; class Stack { public: // Throws std::logic_error if another Stack is alive (see the header comment). explicit Stack(asio::io_context &io, DnsConfig dns_cfg = {}); ~Stack(); Stack(const Stack &) = delete; Stack &operator=(const Stack &) = delete; // Stops the periodic timer, and with it the last thing keeping the // io_context busy. // // The timer re-arms itself for as long as the stack is alive, which makes it // outstanding io work that releasing a work guard cannot retire. The // destructor cancels it -- but the destructor cannot run until run() returns, // and run() will not return while the timer is pending. A process that has // logged a picture-perfect graceful shutdown then sits there forever. So // shutdown has to say so explicitly, here. // // Call it only once every netif is gone: lwIP's timers drive TCP // retransmission and reassembly expiry, so a stack that has stopped ticking // under a live stream stops retransmitting on it. Idempotent. void stop(); const Strand &strand() const { return strand_; } asio::io_context &io() { return io_; } const DnsConfig &dns_config() const { return dns_cfg_; } using AttachHandler = std::function)>; // Brings up a netif over `link` and starts its receive loop. `link` must // outlive the returned Netif; in practice the same object owns both. // // Fails with ovg::Error::ResourceExhausted if `cfg.address` is already in use // by a live netif -- see the address-collision note above. void async_attach(PacketLink &link, NetifConfig cfg, const asio::any_io_executor &cb_ex, AttachHandler h); // lwIP's global counters. Not per-netif -- lwIP does not track them that way // -- so with two tunnels up these are the sum. The health monitor uses the // delta over a window, where that is still a usable signal. // // There is no retransmit counter here because lwIP does not keep one: it // increments tcp.xmit for original and retransmitted segments alike. The // usable proxies for a degrading link are tcp_segments_sent rising while // tcp_segments_received does not, and tcp_memory_errors climbing at all. struct GlobalStats { uint64_t tcp_segments_sent = 0; uint64_t tcp_segments_received = 0; uint64_t tcp_drops = 0; uint64_t tcp_checksum_errors = 0; uint64_t tcp_memory_errors = 0; uint64_t ip_drops = 0; uint64_t ip_checksum_errors = 0; uint64_t reassembly_failures = 0; size_t netifs = 0; }; GlobalStats global_stats() const; private: friend class Netif; void schedule_timer(); void on_timer(const std::error_code &ec); // Address bookkeeping, so collisions are caught before lwIP sees them. // Guarded by mu_ because the admin endpoint reads it off-strand. bool claim_address(const IpAddress &a); void release_address(const IpAddress &a); asio::io_context &io_; DnsConfig dns_cfg_; Strand strand_; asio::steady_timer timer_; bool stopping_ = false; mutable std::mutex mu_; std::vector claimed_; }; // One tunnel's network interface, and the factory for everything that runs on // it. Destroying it removes the lwIP netif and aborts every stream and socket // still bound to it -- which is exactly what has to happen when a drained // egress goes away. class Netif : public std::enable_shared_from_this { public: ~Netif(); Netif(const Netif &) = delete; Netif &operator=(const Netif &) = delete; using ConnectHandler = std::function; using OpenUdpHandler = std::function; // `addr` must be a literal IPv4 address; names are the resolver's job. void async_connect_tcp(const IpAddress &addr, uint16_t port, Millis timeout, const asio::any_io_executor &cb_ex, ConnectHandler h); // Binds an ephemeral port on this netif's address. void async_open_udp(const asio::any_io_executor &cb_ex, OpenUdpHandler h); // Shared, lazily created, backed by this netif's pushed DNS servers with the // configured fallbacks appended. Safe to call from any thread. // // Held here weakly, so the caller owns it -- in practice the egress, for the // tunnel's lifetime. That is not an ownership nicety: the resolver keeps a UDP // socket, the socket keeps this netif alive, and a strong pointer here would // close a reference cycle that no shutdown path could break. Dropping every // reference costs a rebuilt cache on the next call and nothing else. std::shared_ptr resolver(); // Refuses new streams and sockets, aborts the existing ones, removes the lwIP // netif and stops the receive loop. Idempotent. `on_done` runs on the // executor given at attach time once the netif is gone. void shutdown(std::function on_done = {}); bool is_up() const; const IpAddress &address() const { return cfg_.address; } const IpAddress &gateway() const { return cfg_.gateway; } const std::vector &dns_servers() const { return cfg_.dns; } int mtu() const { return cfg_.mtu; } const std::string &label() const { return cfg_.label; } Stack &stack() const { return stack_; } struct Stats { uint64_t rx_packets = 0; uint64_t rx_bytes = 0; uint64_t rx_malformed = 0; // not IPv4, or shorter than an IP header uint64_t rx_dropped = 0; // lwIP refused it (no buffer, bad checksum) uint64_t tx_packets = 0; uint64_t tx_bytes = 0; uint64_t tx_dropped = 0; // link's transmit queue was full uint64_t tcp_opened = 0; uint64_t tcp_failed = 0; int64_t tcp_active = 0; int64_t udp_active = 0; }; Stats stats() const; // netif->output. Public only because lwIP reaches it through a C function // pointer, which cannot be a friend; not part of the interface callers use. // Returns false when the link dropped the packet, which lwIP sees as ERR_MEM // and retries -- the right answer, since a full transmit queue is congestion, // not failure. bool transmit_from_lwip(pbuf *p); private: friend class Stack; friend class LwipTcpStream; friend class LwipUdpSocket; friend class DnsResolver; struct Impl; // holds the lwIP netif; keeps lwip headers out of this file Netif(Stack &stack, PacketLink &link, NetifConfig cfg, asio::any_io_executor cb_ex); // The lwIP netif, for the PCB pinning in lwip_tcp.cpp / lwip_udp.cpp. Only // valid while up_; nullptr afterwards. Strand-only. Declared here rather than // exposing Impl so that lwIP headers stay out of this file. struct netif *lwip_netif(); static std::shared_ptr create(Stack &stack, PacketLink &link, NetifConfig cfg, asio::any_io_executor cb_ex); // Strand-only. bool bring_up(std::string *err); void tear_down(); void arm_receive(); void on_packet(const std::error_code &ec, size_t n); void register_child(const std::shared_ptr &c); void note_tcp_opened(bool ok); void note_tcp_closed(); void note_udp(int delta); Stack &stack_; PacketLink &link_; NetifConfig cfg_; asio::any_io_executor cb_ex_; std::unique_ptr impl_; std::vector rx_buf_; bool up_ = false; bool receiving_ = false; // Live streams and sockets, weakly held so that dropping the caller's // reference still tears the connection down (see stream.h). Used only to // reach them at shutdown. std::vector> children_; mutable std::mutex stats_mu_; Stats stats_; std::mutex resolver_mu_; std::weak_ptr resolver_; // weak on purpose -- see resolver() }; // lwIP err_t -> std::error_code, in one place so the mapping is consistent // between the TCP, UDP and DNS paths. std::error_code lwip_error(int8_t err); namespace detail { // Objects holding lwIP state must be destroyed on the strand, and never from // inside an lwIP callback -- which is exactly what lets those callbacks carry a // raw `this` pointer as their argument. See common/strand_deleter.h; the direct // egress needs the same trick for the same shape of reason. using ovg::StrandDeleter; } // namespace detail } // namespace ovg::netstack