ovgate/src/netstack/lwip_stack.h

// The userspace TCP/IP stack: lwIP lifecycle, one netif per live tunnel.
//
// ---------------------------------------------------------------------------
// Threading
// ---------------------------------------------------------------------------
// lwIP built with NO_SYS=1 has no locking whatsoever. Every call into it and
// every callback out of it must be serialized, so all of it runs on one asio
// strand -- Stack::strand(). Handlers the caller supplies run on the executor
// the caller passed in, never on the strand, so nothing above this module ever
// touches lwIP state.
//
// A strand rather than a dedicated thread: lwIP requires "no concurrency", not
// "the same thread", and a strand gives that plus the happens-before edges,
// without adding a thread or a second io_context to reason about.
//
// ---------------------------------------------------------------------------
// Why there can only be one Stack
// ---------------------------------------------------------------------------
// lwIP's state is file-scope globals: the netif list, tcp_active_pcbs, the
// timeout wheel, the memp pools. There is exactly one lwIP per process and no
// amount of wrapping changes that. The constructor enforces it rather than
// letting a second instance silently corrupt the first.
//
// This matters because make-before-break (docs/ARCHITECTURE.md 5) needs two
// tunnels alive at once. That works: one Stack, two Netifs, and every PCB
// pinned to its netif with tcp_bind_netif()/udp_bind_netif() so routing cannot
// send an old session's packets out the new tunnel.
//
// It works with one exception, and it is worth stating plainly. Attribution of
// *inbound* packets is by destination address, so two tunnels that push the
// same address are indistinguishable to lwIP. VPNGate servers hand out private
// addresses from a small set of ranges, so this is not hypothetical. Netif
// creation therefore rejects an address already in use, and the switch
// controller treats that rejection as "graceful drain is impossible here" and
// falls back to a hard switch -- which is the behaviour the requirements ask
// for when connections cannot be preserved.
//
// ---------------------------------------------------------------------------
// IPv6
// ---------------------------------------------------------------------------
// Not supported. lwIP is compiled IPv4-only (see lwip_port/lwipopts.h): VPNGate
// nodes essentially never push a usable IPv6 prefix, and carrying a second
// address family through the netif, the resolver and the SOCKS5 layer would
// roughly double the surface for something no node exercises. An IPv6 target
// arriving over SOCKS5 is refused with "address type not supported" rather than
// being quietly resolved to something else.
#pragma once

#include <asio.hpp>

#include <cstdint>
#include <functional>
#include <memory>
#include <mutex>
#include <string>
#include <vector>

#include "common/config.h"
#include "common/endpoint.h"
#include "common/strand_deleter.h"
#include "netstack/packet_link.h"
#include "netstack/stream.h"

struct pbuf;
struct netif;

namespace ovg::netstack {

using ovg::Strand;

class Netif;
class LwipTcpStream;
class LwipUdpSocket;
class DnsResolver;

// Implemented by everything a Netif can own. The netif holds these weakly, so
// dropping the caller's reference still tears the connection down; this is only
// the handle it needs to reach survivors when the tunnel goes away underneath
// them. Called on the stack's strand.
class Closable {
 public:
  virtual ~Closable() = default;
  virtual void abort_from_netif(const std::error_code &reason) = 0;
};

// Everything a netif needs, extracted from what the VPN server pushed.
struct NetifConfig {
  IpAddress address;              // our address inside the tunnel
  int prefix = 0;                 // 1..32
  IpAddress gateway;              // may be unset on a point-to-point link
  int mtu = 1500;
  std::vector<IpAddress> dns;     // in server priority order
  std::string label;              // node id; appears in every log line
};

class Stack {
 public:
  // Throws std::logic_error if another Stack is alive (see the header comment).
  explicit Stack(asio::io_context &io, DnsConfig dns_cfg = {});
  ~Stack();

  Stack(const Stack &) = delete;
  Stack &operator=(const Stack &) = delete;

  // Stops the periodic timer, and with it the last thing keeping the
  // io_context busy.
  //
  // The timer re-arms itself for as long as the stack is alive, which makes it
  // outstanding io work that releasing a work guard cannot retire. The
  // destructor cancels it -- but the destructor cannot run until run() returns,
  // and run() will not return while the timer is pending. A process that has
  // logged a picture-perfect graceful shutdown then sits there forever. So
  // shutdown has to say so explicitly, here.
  //
  // Call it only once every netif is gone: lwIP's timers drive TCP
  // retransmission and reassembly expiry, so a stack that has stopped ticking
  // under a live stream stops retransmitting on it. Idempotent.
  void stop();

  const Strand &strand() const { return strand_; }
  asio::io_context &io() { return io_; }
  const DnsConfig &dns_config() const { return dns_cfg_; }

  using AttachHandler =
      std::function<void(const std::error_code &, std::shared_ptr<Netif>)>;

  // Brings up a netif over `link` and starts its receive loop. `link` must
  // outlive the returned Netif; in practice the same object owns both.
  //
  // Fails with ovg::Error::ResourceExhausted if `cfg.address` is already in use
  // by a live netif -- see the address-collision note above.
  void async_attach(PacketLink &link, NetifConfig cfg,
                    const asio::any_io_executor &cb_ex, AttachHandler h);

  // lwIP's global counters. Not per-netif -- lwIP does not track them that way
  // -- so with two tunnels up these are the sum. The health monitor uses the
  // delta over a window, where that is still a usable signal.
  //
  // There is no retransmit counter here because lwIP does not keep one: it
  // increments tcp.xmit for original and retransmitted segments alike. The
  // usable proxies for a degrading link are tcp_segments_sent rising while
  // tcp_segments_received does not, and tcp_memory_errors climbing at all.
  struct GlobalStats {
    uint64_t tcp_segments_sent = 0;
    uint64_t tcp_segments_received = 0;
    uint64_t tcp_drops = 0;
    uint64_t tcp_checksum_errors = 0;
    uint64_t tcp_memory_errors = 0;
    uint64_t ip_drops = 0;
    uint64_t ip_checksum_errors = 0;
    uint64_t reassembly_failures = 0;
    size_t netifs = 0;
  };
  GlobalStats global_stats() const;

 private:
  friend class Netif;

  void schedule_timer();
  void on_timer(const std::error_code &ec);

  // Address bookkeeping, so collisions are caught before lwIP sees them.
  // Guarded by mu_ because the admin endpoint reads it off-strand.
  bool claim_address(const IpAddress &a);
  void release_address(const IpAddress &a);

  asio::io_context &io_;
  DnsConfig dns_cfg_;
  Strand strand_;
  asio::steady_timer timer_;
  bool stopping_ = false;

  mutable std::mutex mu_;
  std::vector<IpAddress> claimed_;
};

// One tunnel's network interface, and the factory for everything that runs on
// it. Destroying it removes the lwIP netif and aborts every stream and socket
// still bound to it -- which is exactly what has to happen when a drained
// egress goes away.
class Netif : public std::enable_shared_from_this<Netif> {
 public:
  ~Netif();

  Netif(const Netif &) = delete;
  Netif &operator=(const Netif &) = delete;

  using ConnectHandler =
      std::function<void(const std::error_code &, TcpStreamPtr)>;
  using OpenUdpHandler =
      std::function<void(const std::error_code &, UdpSocketPtr)>;

  // `addr` must be a literal IPv4 address; names are the resolver's job.
  void async_connect_tcp(const IpAddress &addr, uint16_t port, Millis timeout,
                         const asio::any_io_executor &cb_ex, ConnectHandler h);

  // Binds an ephemeral port on this netif's address.
  void async_open_udp(const asio::any_io_executor &cb_ex, OpenUdpHandler h);

  // Shared, lazily created, backed by this netif's pushed DNS servers with the
  // configured fallbacks appended. Safe to call from any thread.
  //
  // Held here weakly, so the caller owns it -- in practice the egress, for the
  // tunnel's lifetime. That is not an ownership nicety: the resolver keeps a UDP
  // socket, the socket keeps this netif alive, and a strong pointer here would
  // close a reference cycle that no shutdown path could break. Dropping every
  // reference costs a rebuilt cache on the next call and nothing else.
  std::shared_ptr<Resolver> resolver();

  // Refuses new streams and sockets, aborts the existing ones, removes the lwIP
  // netif and stops the receive loop. Idempotent. `on_done` runs on the
  // executor given at attach time once the netif is gone.
  void shutdown(std::function<void()> on_done = {});
  bool is_up() const;

  const IpAddress &address() const { return cfg_.address; }
  const IpAddress &gateway() const { return cfg_.gateway; }
  const std::vector<IpAddress> &dns_servers() const { return cfg_.dns; }
  int mtu() const { return cfg_.mtu; }
  const std::string &label() const { return cfg_.label; }
  Stack &stack() const { return stack_; }

  struct Stats {
    uint64_t rx_packets = 0;
    uint64_t rx_bytes = 0;
    uint64_t rx_malformed = 0;   // not IPv4, or shorter than an IP header
    uint64_t rx_dropped = 0;     // lwIP refused it (no buffer, bad checksum)
    uint64_t tx_packets = 0;
    uint64_t tx_bytes = 0;
    uint64_t tx_dropped = 0;     // link's transmit queue was full
    uint64_t tcp_opened = 0;
    uint64_t tcp_failed = 0;
    int64_t tcp_active = 0;
    int64_t udp_active = 0;
  };
  Stats stats() const;

  // netif->output. Public only because lwIP reaches it through a C function
  // pointer, which cannot be a friend; not part of the interface callers use.
  // Returns false when the link dropped the packet, which lwIP sees as ERR_MEM
  // and retries -- the right answer, since a full transmit queue is congestion,
  // not failure.
  bool transmit_from_lwip(pbuf *p);

 private:
  friend class Stack;
  friend class LwipTcpStream;
  friend class LwipUdpSocket;
  friend class DnsResolver;

  struct Impl;  // holds the lwIP netif; keeps lwip headers out of this file

  Netif(Stack &stack, PacketLink &link, NetifConfig cfg,
        asio::any_io_executor cb_ex);

  // The lwIP netif, for the PCB pinning in lwip_tcp.cpp / lwip_udp.cpp. Only
  // valid while up_; nullptr afterwards. Strand-only. Declared here rather than
  // exposing Impl so that lwIP headers stay out of this file.
  struct netif *lwip_netif();

  static std::shared_ptr<Netif> create(Stack &stack, PacketLink &link,
                                       NetifConfig cfg,
                                       asio::any_io_executor cb_ex);

  // Strand-only.
  bool bring_up(std::string *err);
  void tear_down();
  void arm_receive();
  void on_packet(const std::error_code &ec, size_t n);
  void register_child(const std::shared_ptr<Closable> &c);
  void note_tcp_opened(bool ok);
  void note_tcp_closed();
  void note_udp(int delta);

  Stack &stack_;
  PacketLink &link_;
  NetifConfig cfg_;
  asio::any_io_executor cb_ex_;
  std::unique_ptr<Impl> impl_;

  std::vector<uint8_t> rx_buf_;
  bool up_ = false;
  bool receiving_ = false;

  // Live streams and sockets, weakly held so that dropping the caller's
  // reference still tears the connection down (see stream.h). Used only to
  // reach them at shutdown.
  std::vector<std::weak_ptr<Closable>> children_;

  mutable std::mutex stats_mu_;
  Stats stats_;

  std::mutex resolver_mu_;
  std::weak_ptr<Resolver> resolver_;  // weak on purpose -- see resolver()
};

// lwIP err_t -> std::error_code, in one place so the mapping is consistent
// between the TCP, UDP and DNS paths.
std::error_code lwip_error(int8_t err);

namespace detail {

// Objects holding lwIP state must be destroyed on the strand, and never from
// inside an lwIP callback -- which is exactly what lets those callbacks carry a
// raw `this` pointer as their argument. See common/strand_deleter.h; the direct
// egress needs the same trick for the same shape of reason.
using ovg::StrandDeleter;

}  // namespace detail

}  // namespace ovg::netstack