// L2 health checking: is the *current* egress still worth keeping? // // --------------------------------------------------------------------------- // Two layers, and this is only the second // --------------------------------------------------------------------------- // L1 is openvpn3's own ping/ping-restart reconnect. It happens inside the // session, the tun fd survives it (tunPersist), and lwIP never notices -- so it // costs nothing and covers the overwhelming majority of transient trouble. Do // not duplicate it here. // // L2 is "this node is not coming back, find another one". It is expensive: a // switch drops every session that has already moved bytes. So the bar has to be // high, and it has to be *sustained* -- `unhealthy_windows` consecutive bad // samples, not one. A single 5s blip on a volunteer-run VPN in another country // is normal weather (docs/ARCHITECTURE.md ยง7). // // --------------------------------------------------------------------------- // Why the probe is a TCP handshake and not a DNS lookup // --------------------------------------------------------------------------- // The obvious in-tunnel RTT probe is a timed DNS query. It is also wrong: both // resolvers we ship cache, so the second query onwards is answered from memory // in ~0ms without a single byte crossing the tunnel. That reports a *dead* // tunnel as the healthiest thing in the fleet -- the exact failure the monitor // exists to catch. // // So the probe dials `probe_domain:probe_port` through the egress and drops the // stream the moment it is up. A handshake cannot be served from a cache, it // exercises the same path a real session uses, and its timing is a genuine // round trip. The name lookup still happens inside it, so a broken in-tunnel // resolver still shows up -- as a failed probe rather than a slow one. // // --------------------------------------------------------------------------- // Scoring, and what happens when a signal is not available // --------------------------------------------------------------------------- // Four weighted terms in [0,1], plus one veto: // // veto egress absent or not Ready -> score 0, no probe attempted // rtt probe latency, 0ms..1000ms -> 0.35 // connect SOCKS5 CONNECT success rate -> 0.30 // stall active sessions but no byte motion -> 0.20 // loss netstack drop counters -> 0.15 // // The stall term needs byte counters the direct egress does not keep (it has no // single aggregate to report, egress/egress.h). Rather than let a permanent // zero read as a permanent stall, a term whose input is unavailable is dropped // and the remaining weights are renormalised. A missing signal must not be // scored as a bad one. // // --------------------------------------------------------------------------- // Threading // --------------------------------------------------------------------------- // Everything runs on an internal strand. `last()`/`recent()` take a short mutex // and are callable from the admin thread. The unhealthy handler is invoked on // the strand; SwitchController forwards it straight to EgressManager, which has // its own. #pragma once #include #include #include #include #include #include #include #include #include #include "common/config.h" #include "common/strand_deleter.h" // ovg::Strand #include "egress/egress.h" namespace ovg::health { using Clock = std::chrono::steady_clock; // One sampling round. Kept whole (rather than reduced to a bool) because "why // did it decide that" is the only question anyone asks of a health check. struct Sample { int64_t age_ms = 0; // how long ago this was taken, filled in on read std::string egress_label; bool egress_present = false; bool tunnel_up = false; double rtt_ms = -1.0; // -1 = probe did not complete bool probe_ok = false; double connect_failure_rate = 0.0; int64_t stalled_ms = 0; // 0 = moving, or unmeasurable bool stall_known = false; // false = egress keeps no byte counters double loss_rate = 0.0; double score = 0.0; bool healthy = false; std::string verdict; // one line, human-first Clock::time_point at{}; }; class HealthMonitor { public: // Returns the egress to sample, or null if there is none. Same seam as // socks5::Server::EgressProvider, and for the same reason: the monitor is // testable against a DirectEgress with no manager in sight. using EgressProvider = std::function; using UnhealthyHandler = std::function; HealthMonitor(asio::io_context &io, HealthConfig cfg, EgressProvider acquire); ~HealthMonitor(); HealthMonitor(const HealthMonitor &) = delete; HealthMonitor &operator=(const HealthMonitor &) = delete; // Fires once per *sustained* failure, not once per bad sample: after it fires // the window counter resets, so a node that stays bad does not produce a // switch request every `interval`. Set before start(). void set_on_unhealthy(UnhealthyHandler h); void start(); void stop(); // Runs a round now, off-schedule, and re-arms the timer from here. Used by // the admin endpoint and by SwitchController right after a promotion, when // waiting a full interval to learn whether the new node works is too slow. void probe_now(); Sample last() const; std::vector recent(size_t n) const; int consecutive_bad() const; uint64_t rounds() const; private: void arm(); void run_round(); void finish_probe(egress::EgressPtr eg, double rtt_ms, bool ok, const std::string &probe_note); void publish(Sample s); // Byte-motion bookkeeping, reset whenever the egress underneath changes. void update_stall(const egress::EgressStats &st, Sample *out); asio::io_context &io_; HealthConfig cfg_; EgressProvider acquire_; Strand strand_; asio::steady_timer timer_; asio::steady_timer probe_timer_; UnhealthyHandler on_unhealthy_; std::atomic running_{false}; // Strand-only. bool probe_in_flight_ = false; std::string stall_label_; uint64_t stall_bytes_ = 0; Clock::time_point stall_since_{}; bool traffic_ever_seen_ = false; mutable std::mutex mu_; std::deque history_; int consecutive_bad_ = 0; uint64_t rounds_ = 0; }; } // namespace ovg::health