ovgate/tests/test_csv_parser.cpp

// The VPNGate feed is hostile in a boring way: very long lines, volunteer-typed
// free text, and occasional rows that are simply broken. These tests pin the
// two behaviours we actually depend on -- no line-length assumptions, and one
// bad row never costs us the rest of the list.
#include <fstream>
#include <sstream>

#include "harness.h"
#include "vpngate/csv_parser.h"

using namespace ovg::vpngate;

namespace {

std::string read_sample() {
  std::ifstream in(ovgtest::data_path("vpngate_sample.csv"), std::ios::binary);
  CHECK(in.good());
  std::ostringstream ss;
  ss << in.rdbuf();
  return ss.str();
}

}  // namespace

OVG_TEST(SplitCsvBasic) {
  auto f = split_csv_line("a,b,c");
  CHECK_EQ(f.size(), size_t(3));
  CHECK_EQ(f[0], std::string("a"));
  CHECK_EQ(f[2], std::string("c"));
}

OVG_TEST(SplitCsvEmptyFields) {
  auto f = split_csv_line("a,,c,");
  CHECK_EQ(f.size(), size_t(4));
  CHECK_EQ(f[1], std::string(""));
  CHECK_EQ(f[3], std::string(""));
}

OVG_TEST(SplitCsvQuotedCommaAndDoubledQuote) {
  auto f = split_csv_line(R"(a,"b,still b","he said ""hi""",d)");
  CHECK_EQ(f.size(), size_t(4));
  CHECK_EQ(f[1], std::string("b,still b"));
  CHECK_EQ(f[2], std::string("he said \"hi\""));
  CHECK_EQ(f[3], std::string("d"));
}

OVG_TEST(SplitCsvStripsTrailingCr) {
  auto f = split_csv_line("a,b\r");
  CHECK_EQ(f.size(), size_t(2));
  CHECK_EQ(f[1], std::string("b"));
}

OVG_TEST(SplitCsvToleratesStrayQuote) {
  // Volunteers type things like: Operator: 5" floppy fan club
  auto f = split_csv_line("a,5\" floppy,c");
  CHECK_EQ(f.size(), size_t(3));
  CHECK_EQ(f[2], std::string("c"));
}

OVG_TEST(Base64RoundTrip) {
  std::string out;
  CHECK(base64_decode("aGVsbG8gd29ybGQ=", &out));
  CHECK_EQ(out, std::string("hello world"));

  CHECK(base64_decode("", &out));
  CHECK_EQ(out, std::string(""));

  // Embedded newlines are common in the wild; they must be ignored, not fatal.
  CHECK(base64_decode("aGVs\nbG8g\nd29ybGQ=", &out));
  CHECK_EQ(out, std::string("hello world"));
}

OVG_TEST(Base64RejectsGarbage) {
  std::string out;
  CHECK(!base64_decode("not*valid*base64", &out));
}

OVG_TEST(ExtractRemotesHonoursProtoRegardlessOfOrder) {
  // "proto" appears *after* the remotes here. A single-pass parser would
  // mislabel both as the default TCP.
  const char *profile =
      "client\n"
      "dev tun\n"
      "remote 1.2.3.4 1194\n"
      "remote 1.2.3.4 443\n"
      "proto udp\n"
      "resolv-retry infinite\n";
  auto r = extract_remotes(profile);
  CHECK_EQ(r.size(), size_t(2));
  CHECK(r[0].proto == Proto::Udp);
  CHECK(r[1].proto == Proto::Udp);
  CHECK_EQ(r[0].port, uint16_t(1194));
  CHECK_EQ(r[1].port, uint16_t(443));
}

OVG_TEST(ExtractRemotesPerRemoteProtoWins) {
  const char *profile =
      "proto tcp\n"
      "remote 1.2.3.4 1194 udp\n"
      "remote 5.6.7.8 443\n";
  auto r = extract_remotes(profile);
  CHECK_EQ(r.size(), size_t(2));
  CHECK(r[0].proto == Proto::Udp);
  CHECK(r[1].proto == Proto::Tcp);
}

OVG_TEST(ExtractRemotesUnderstandsTcpClient) {
  const char *profile = "proto tcp-client\nremote 1.2.3.4 443\n";
  auto r = extract_remotes(profile);
  CHECK_EQ(r.size(), size_t(1));
  CHECK(r[0].proto == Proto::Tcp);
}

OVG_TEST(ExtractRemotesAppliesBarePortDirective) {
  const char *profile = "port 1194\nremote 1.2.3.4\nproto udp\n";
  auto r = extract_remotes(profile);
  CHECK_EQ(r.size(), size_t(1));
  CHECK_EQ(r[0].port, uint16_t(1194));
}

OVG_TEST(ExtractRemotesIgnoresInlineBlocks) {
  // Two traps. A <connection> block declares an alternative remote we never
  // scored -- and the sanitizer strips those blocks outright, so counting it
  // here would make the node list and the profile we actually dial disagree.
  // A PEM body is arbitrary base64 that can start a line with any word.
  const char *profile =
      "remote 1.2.3.4 443 tcp\n"
      "<connection>\n"
      "remote 9.9.9.9 1194 udp\n"
      "</connection>\n"
      "<ca>\n"
      "remote 8.8.8.8 53 udp\n"
      "proto udp\n"
      "-----END CERTIFICATE-----\n"
      "</ca>\n";
  auto r = extract_remotes(profile);
  CHECK_EQ(r.size(), size_t(1));
  CHECK_EQ(r[0].host, std::string("1.2.3.4"));
  CHECK(r[0].proto == Proto::Tcp);
}

OVG_TEST(ExtractRemotesMatchesBlockTagsCaseInsensitively) {
  const char *profile =
      "remote 1.2.3.4 443 tcp\n"
      "<CA>\n"
      "remote 9.9.9.9 1194 udp\n"
      "</ca>\n"
      "remote 5.6.7.8 443 tcp\n";
  auto r = extract_remotes(profile);
  CHECK_EQ(r.size(), size_t(2));
  CHECK_EQ(r[1].host, std::string("5.6.7.8"));
}

OVG_TEST(ParseRejectsHtmlErrorPage) {
  ParseResult res;
  std::string err;
  CHECK(!parse_node_list("<html><body>503</body></html>", &res, &err));
  CHECK(!err.empty());
}

OVG_TEST(ParseRejectsMissingMagic) {
  ParseResult res;
  std::string err;
  CHECK(!parse_node_list("#HostName,IP\nfoo,1.2.3.4\n", &res, &err));
}

OVG_TEST(ParseSkipsBadRowsAndKeepsGoing) {
  // Row 2 is truncated, row 3 has undecodable base64. Row 1 and 4 must survive.
  std::string good_profile_b64;
  {
    // "client\nremote 1.2.3.4 443 tcp\n" base64-encoded.
    good_profile_b64 = "Y2xpZW50CnJlbW90ZSAxLjIuMy40IDQ0MyB0Y3AK";
  }
  std::ostringstream body;
  body << "*vpn_servers\n"
       << "#HostName,IP,Score,Ping,Speed,CountryLong,CountryShort,"
          "NumVpnSessions,Uptime,TotalUsers,TotalTraffic,LogType,Operator,"
          "Message,OpenVPN_ConfigData_Base64\n"
       << "ok1,1.1.1.1,100,10,1000,Japan,JP,1,1000,1,1,2weeks,op,,"
       << good_profile_b64 << "\n"
       << "truncated,2.2.2.2,100\n"
       << "badb64,3.3.3.3,100,10,1000,Japan,JP,1,1000,1,1,2weeks,op,,!!!!\n"
       << "ok2,4.4.4.4,200,20,2000,Korea,KR,2,2000,2,2,2weeks,op,,"
       << good_profile_b64 << "\n"
       << "*\n";

  ParseResult res;
  std::string err;
  CHECK(parse_node_list(body.str(), &res, &err));
  CHECK_EQ(res.nodes.size(), size_t(2));
  CHECK_EQ(res.nodes[0].host_name, std::string("ok1"));
  CHECK_EQ(res.nodes[1].host_name, std::string("ok2"));
  CHECK_EQ(res.stats.data_rows, size_t(4));
  CHECK_EQ(res.stats.accepted, size_t(2));
  CHECK_EQ(res.stats.skipped_columns, size_t(1));
  CHECK_EQ(res.stats.skipped_base64, size_t(1));
}

OVG_TEST(ParseRealSampleFeed) {
  const auto body = read_sample();
  CHECK_GT(body.size(), size_t(1000000));

  ParseResult res;
  std::string err;
  CHECK(parse_node_list(body, &res, &err));

  // The captured feed had 96 data rows; every one of them should parse.
  CHECK_GT(res.nodes.size(), size_t(80));
  CHECK_EQ(res.stats.accepted, res.stats.data_rows);

  for (const auto &n : res.nodes) {
    CHECK(!n.host_name.empty());
    CHECK(!n.ip.empty());
    CHECK(!n.remotes.empty());
    // openvpn3 needs the whole profile, inline certs and all.
    CHECK_GT(n.profile.size(), size_t(1000));
    CHECK_NE(n.profile.find("<ca>"), std::string::npos);
    CHECK_NE(n.id().find('@'), std::string::npos);
  }
}

OVG_TEST(RealSampleHasVeryLongLines) {
  // Guards the requirement explicitly: if this ever fits in a 4 KB buffer, the
  // test data stopped being representative.
  const auto body = read_sample();
  size_t longest = 0, start = 0;
  for (size_t i = 0; i <= body.size(); ++i) {
    if (i == body.size() || body[i] == '\n') {
      longest = std::max(longest, i - start);
      start = i + 1;
    }
  }
  CHECK_GT(longest, size_t(8192));
}

OVG_TEST(RealSampleNodesOfferTcp443) {
  // The tunnel design assumes almost every node exposes TCP; the prober cannot
  // time a UDP-only node (see selector/prober.h).
  const auto body = read_sample();
  ParseResult res;
  std::string err;
  CHECK(parse_node_list(body, &res, &err));

  size_t with_tcp = 0;
  for (const auto &n : res.nodes)
    if (n.has_tcp()) ++with_tcp;
  CHECK_GT(with_tcp, res.nodes.size() * 9 / 10);
}