ovgate/src/health/health_monitor.h

// L2 health checking: is the *current* egress still worth keeping?
//
// ---------------------------------------------------------------------------
// Two layers, and this is only the second
// ---------------------------------------------------------------------------
// L1 is openvpn3's own ping/ping-restart reconnect. It happens inside the
// session, the tun fd survives it (tunPersist), and lwIP never notices -- so it
// costs nothing and covers the overwhelming majority of transient trouble. Do
// not duplicate it here.
//
// L2 is "this node is not coming back, find another one". It is expensive: a
// switch drops every session that has already moved bytes. So the bar has to be
// high, and it has to be *sustained* -- `unhealthy_windows` consecutive bad
// samples, not one. A single 5s blip on a volunteer-run VPN in another country
// is normal weather (docs/ARCHITECTURE.md §7).
//
// ---------------------------------------------------------------------------
// Why the probe is a TCP handshake and not a DNS lookup
// ---------------------------------------------------------------------------
// The obvious in-tunnel RTT probe is a timed DNS query. It is also wrong: both
// resolvers we ship cache, so the second query onwards is answered from memory
// in ~0ms without a single byte crossing the tunnel. That reports a *dead*
// tunnel as the healthiest thing in the fleet -- the exact failure the monitor
// exists to catch.
//
// So the probe dials `probe_domain:probe_port` through the egress and drops the
// stream the moment it is up. A handshake cannot be served from a cache, it
// exercises the same path a real session uses, and its timing is a genuine
// round trip. The name lookup still happens inside it, so a broken in-tunnel
// resolver still shows up -- as a failed probe rather than a slow one.
//
// ---------------------------------------------------------------------------
// Scoring, and what happens when a signal is not available
// ---------------------------------------------------------------------------
// Four weighted terms in [0,1], plus one veto:
//
//   veto    egress absent or not Ready          -> score 0, no probe attempted
//   rtt     probe latency, 0ms..1000ms          -> 0.35
//   connect SOCKS5 CONNECT success rate         -> 0.30
//   stall   active sessions but no byte motion  -> 0.20
//   loss    netstack drop counters              -> 0.15
//
// The stall term needs byte counters the direct egress does not keep (it has no
// single aggregate to report, egress/egress.h). Rather than let a permanent
// zero read as a permanent stall, a term whose input is unavailable is dropped
// and the remaining weights are renormalised. A missing signal must not be
// scored as a bad one.
//
// ---------------------------------------------------------------------------
// Threading
// ---------------------------------------------------------------------------
// Everything runs on an internal strand. `last()`/`recent()` take a short mutex
// and are callable from the admin thread. The unhealthy handler is invoked on
// the strand; SwitchController forwards it straight to EgressManager, which has
// its own.
#pragma once

#include <asio.hpp>

#include <atomic>
#include <chrono>
#include <cstdint>
#include <deque>
#include <functional>
#include <mutex>
#include <string>
#include <vector>

#include "common/config.h"
#include "common/strand_deleter.h"  // ovg::Strand
#include "egress/egress.h"

namespace ovg::health {

using Clock = std::chrono::steady_clock;

// One sampling round. Kept whole (rather than reduced to a bool) because "why
// did it decide that" is the only question anyone asks of a health check.
struct Sample {
  int64_t age_ms = 0;  // how long ago this was taken, filled in on read
  std::string egress_label;
  bool egress_present = false;
  bool tunnel_up = false;

  double rtt_ms = -1.0;  // -1 = probe did not complete
  bool probe_ok = false;
  double connect_failure_rate = 0.0;
  int64_t stalled_ms = 0;   // 0 = moving, or unmeasurable
  bool stall_known = false; // false = egress keeps no byte counters
  double loss_rate = 0.0;

  double score = 0.0;
  bool healthy = false;
  std::string verdict;  // one line, human-first

  Clock::time_point at{};
};

class HealthMonitor {
 public:
  // Returns the egress to sample, or null if there is none. Same seam as
  // socks5::Server::EgressProvider, and for the same reason: the monitor is
  // testable against a DirectEgress with no manager in sight.
  using EgressProvider = std::function<egress::EgressPtr()>;
  using UnhealthyHandler = std::function<void(const std::string &why)>;

  HealthMonitor(asio::io_context &io, HealthConfig cfg, EgressProvider acquire);
  ~HealthMonitor();

  HealthMonitor(const HealthMonitor &) = delete;
  HealthMonitor &operator=(const HealthMonitor &) = delete;

  // Fires once per *sustained* failure, not once per bad sample: after it fires
  // the window counter resets, so a node that stays bad does not produce a
  // switch request every `interval`. Set before start().
  void set_on_unhealthy(UnhealthyHandler h);

  void start();
  void stop();

  // Runs a round now, off-schedule, and re-arms the timer from here. Used by
  // the admin endpoint and by SwitchController right after a promotion, when
  // waiting a full interval to learn whether the new node works is too slow.
  void probe_now();

  Sample last() const;
  std::vector<Sample> recent(size_t n) const;
  int consecutive_bad() const;
  uint64_t rounds() const;

 private:
  void arm();
  void run_round();
  void finish_probe(egress::EgressPtr eg, double rtt_ms, bool ok,
                    const std::string &probe_note);
  void publish(Sample s);
  // Byte-motion bookkeeping, reset whenever the egress underneath changes.
  void update_stall(const egress::EgressStats &st, Sample *out);

  asio::io_context &io_;
  HealthConfig cfg_;
  EgressProvider acquire_;
  Strand strand_;
  asio::steady_timer timer_;
  asio::steady_timer probe_timer_;

  UnhealthyHandler on_unhealthy_;
  std::atomic<bool> running_{false};

  // Strand-only.
  bool probe_in_flight_ = false;
  std::string stall_label_;
  uint64_t stall_bytes_ = 0;
  Clock::time_point stall_since_{};
  bool traffic_ever_seen_ = false;

  mutable std::mutex mu_;
  std::deque<Sample> history_;
  int consecutive_bad_ = 0;
  uint64_t rounds_ = 0;
};

}  // namespace ovg::health