Skip to content

Commit afcfbf3

Browse files
committed
Add SEIP server discovery: <id>.<region>.seip.<domain> + NS slot claiming
Major discovery redesign for scalable, region-aware server selection across hundreds of community servers. DNS hierarchy: - Servers register as <id>.<region>.seip.lemonade-nexus.io - _config TXT includes region + load (connected client count) - First 9 servers claim ns1-ns9 via democratic gossip (LWW tiebreak) - NS glue records served by our authoritative DNS, cached globally Server changes: - ServerConfig: add --region / SP_REGION (auto-detected via geo-IP) - resolve_server_region(): config > persisted > auto-detect - GossipService: NsSlotClaim (0x14) message type, try_claim_ns_slot() - GossipPeer: region field, included in ServerHello - DnsService: publish_seip_records(), update_load(), set_server_region() - Multi-answer A responses for region wildcards (capped at 5) - _config TXT now includes region= and load= fields Client changes (Swift): - Region-aware discovery: detect own region via ip-api.com geo lookup - Query <region>.seip.<domain> for servers in same region first - Fallback to adjacent regions ordered by geographic distance - Score = latency_ms + (load * 10) for load-aware server selection - Backward-compatible legacy discovery as final fallback
1 parent e0e403d commit afcfbf3

File tree

14 files changed

+1018
-56
lines changed

14 files changed

+1018
-56
lines changed

apps/LemonadeNexusMac/Sources/LemonadeNexusMac/Services/DnsDiscovery.swift

Lines changed: 347 additions & 48 deletions
Large diffs are not rendered by default.
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
---
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Start testing: Mar 17 22:30 PDT
2+
----------------------------------------------------------
3+
End testing: Mar 17 22:30 PDT

projects/LemonadeNexus/include/LemonadeNexus/Core/ServerConfig.hpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ struct ServerConfig {
1818
uint16_t dns_port{53};
1919
std::string bind_address{"0.0.0.0"};
2020
std::string public_ip; // public-facing IP for DNS glue records (auto-detected if empty)
21+
std::string region; // cloud region code (e.g. "us-east", auto-detected if empty)
2122

2223
// Storage
2324
std::string data_root{"data"};

projects/LemonadeNexus/include/LemonadeNexus/Core/ServerIdentity.hpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,12 @@ void resolve_server_hostname(
3636
const std::filesystem::path& data_root,
3737
gossip::GossipService& gossip);
3838

39+
/// Resolve server region: config > persisted > auto-detect via HostnameGenerator.
40+
/// Mutates config.region in-place and persists to data/identity/region.
41+
void resolve_server_region(
42+
ServerConfig& config,
43+
const std::filesystem::path& data_root);
44+
3945
/// Resolve public IP: config > non-wildcard bind address > ipify auto-detect.
4046
[[nodiscard]] std::string resolve_public_ip(const ServerConfig& config);
4147

projects/LemonadeNexus/include/LemonadeNexus/Gossip/GossipService.hpp

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,21 @@ class GossipService : public core::IService<GossipService>,
7979
/// Broadcast a backbone IPAM allocation delta to all peers.
8080
void broadcast_backbone_ipam_delta(const ipam::BackboneAllocationDelta& delta);
8181

82+
/// Set the cloud region code for this server (e.g. "us-east-1").
83+
void set_our_region(const std::string& region);
84+
85+
/// Set the DNS base domain for NS slot FQDN construction (default: "lemonade-nexus.io").
86+
void set_dns_base_domain(const std::string& domain);
87+
88+
/// Attempt to claim the lowest available NS slot (ns1-ns9) via gossip.
89+
void try_claim_ns_slot(const std::string& our_public_ip);
90+
91+
/// Returns our claimed NS slot number (1-9), or nullopt if we don't hold one.
92+
[[nodiscard]] std::optional<uint8_t> our_ns_slot() const;
93+
94+
/// Returns all currently claimed NS slots (for status reporting).
95+
[[nodiscard]] std::vector<NsSlotClaimData> get_ns_slots() const;
96+
8297
/// Try to add a gossip peer as a WireGuard backbone peer.
8398
void try_add_backbone_wg_peer(const GossipPeer& peer);
8499

@@ -227,6 +242,16 @@ class GossipService : public core::IService<GossipService>,
227242
void handle_backbone_ipam_sync(const asio::ip::udp::endpoint& sender,
228243
const uint8_t* payload, std::size_t payload_len);
229244

245+
// NS slot claim handler
246+
void handle_ns_slot_claim(const asio::ip::udp::endpoint& sender,
247+
const uint8_t* payload, std::size_t payload_len);
248+
249+
/// Broadcast an NS slot claim to all known peers.
250+
void broadcast_ns_slot_claim(const NsSlotClaimData& claim);
251+
252+
/// Register an NS slot claim in the local DNS service (if available).
253+
void register_ns_slot_in_dns(const NsSlotClaimData& claim);
254+
230255
// Verify a server certificate against the root pubkey
231256
[[nodiscard]] bool verify_server_certificate(const ServerCertificate& cert) const;
232257

@@ -281,6 +306,12 @@ class GossipService : public core::IService<GossipService>,
281306
std::vector<std::string> reconstruction_shares_;
282307
uint8_t reconstruction_threshold_{0};
283308

309+
// Democratic NS slot claiming (ns1-ns9 bootstrap nameservers)
310+
std::string our_region_;
311+
std::string dns_base_domain_{"lemonade-nexus.io"};
312+
std::array<NsSlotClaimData, 9> ns_slots_{}; // slot 0 = ns1, slot 8 = ns9
313+
std::optional<uint8_t> our_ns_slot_;
314+
284315
// Distributed ACL sync (nullptr = ACL sync disabled)
285316
acl::ACLService* acl_{nullptr};
286317

projects/LemonadeNexus/include/LemonadeNexus/Gossip/GossipTypes.hpp

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ enum class GossipMsgType : uint8_t {
3232
AclDelta = 0x11, // "ACL grant/revoke — distributed permission sync"
3333
DnsRecordSync = 0x12, // "DNS record add/remove — distributed authoritative DNS"
3434
BackboneIpamSync = 0x13, // "backbone IP allocate/release — server mesh IPAM sync"
35+
NsSlotClaim = 0x14, // "democratic NS slot claim — ns1-ns9 bootstrap nameservers"
3536
};
3637

3738
#pragma pack(push, 1)
@@ -54,6 +55,7 @@ struct GossipPeer {
5455
std::string backbone_endpoint; // "ip:port" (gossip port, over WG backbone — preferred when available)
5556
std::string wg_pubkey; // base64 X25519 WireGuard public key
5657
std::string backbone_ip; // "172.16.0.X" (empty until allocated)
58+
std::string region; // cloud region code (e.g. "us-east-1")
5759
uint16_t http_port{9100}; // HTTP control plane port
5860
uint64_t last_seen{0}; // Unix timestamp
5961
float reputation{1.0f};
@@ -185,4 +187,19 @@ struct DnsRecordDelta {
185187
std::string signature; // Ed25519 over canonical JSON (excludes this field)
186188
};
187189

190+
// ---------------------------------------------------------------------------
191+
// Democratic NS slot claiming (ns1-ns9 bootstrap nameservers)
192+
// ---------------------------------------------------------------------------
193+
194+
/// A signed NS slot claim that propagates via gossip.
195+
/// The first 9 servers to join the mesh claim ns1-ns9 slots (LWW conflict resolution).
196+
struct NsSlotClaimData {
197+
uint8_t slot{0}; // 1-9
198+
std::string server_pubkey; // base64 Ed25519
199+
std::string server_ip; // public IP
200+
std::string region; // cloud region code
201+
uint64_t timestamp{0}; // LWW conflict resolution
202+
std::string signature; // Ed25519 signature
203+
};
204+
188205
} // namespace nexus::gossip

projects/LemonadeNexus/include/LemonadeNexus/Network/DnsService.hpp

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,11 @@ using DnsRecordCallback = std::function<void(const std::string& delta_id,
5454
/// <hostname>.<base_domain> -> Any node matching the hostname
5555
/// <hostname>.<region>.relays.<base_domain> -> Relay in specific region
5656
/// <hostname>.relays.<base_domain> -> Relay by hostname (any region)
57-
/// _config.<hostname>.<base_domain> -> TXT record with port config
58-
/// _acme-challenge.<domain> -> ACME DNS-01 TXT challenge
57+
/// <id>.<region>.seip.<base_domain> -> Server's public IP (SEIP A record)
58+
/// <region>.seip.<base_domain> -> All servers in region (multi-A)
59+
/// _config.<id>.<region>.seip.<base_domain> -> Server config TXT (SEIP)
60+
/// _config.<hostname>.<base_domain> -> TXT record with port config
61+
/// _acme-challenge.<domain> -> ACME DNS-01 TXT challenge
5962
class DnsService : public core::IService<DnsService>,
6063
public IDnsProvider<DnsService> {
6164
friend class core::IService<DnsService>;
@@ -150,6 +153,8 @@ class DnsService : public core::IService<DnsService>,
150153
uint16_t relay_port{9103};
151154
uint16_t dns_port{53};
152155
uint16_t private_http_port{9101};
156+
std::string region;
157+
uint32_t connected_clients{0};
153158
};
154159

155160
/// Set the port configuration to publish via TXT records.
@@ -164,6 +169,22 @@ class DnsService : public core::IService<DnsService>,
164169
[[nodiscard]] std::optional<std::string> resolve_config_txt(
165170
const std::string& hostname);
166171

172+
// -----------------------------------------------------------------
173+
// SEIP DNS record hierarchy
174+
// -----------------------------------------------------------------
175+
176+
/// Set the region for this server (used in SEIP record publishing).
177+
void set_server_region(const std::string& region);
178+
179+
/// Publish SEIP A and _config TXT records under <id>.<region>.seip.<domain>.
180+
void publish_seip_records(const std::string& server_id,
181+
const std::string& region,
182+
const std::string& public_ip);
183+
184+
/// Update the connected-client load count and re-publish the SEIP _config
185+
/// TXT record if the count changed (avoids gossip spam).
186+
void update_load(uint32_t connected_client_count);
187+
167188
private:
168189
void start_receive();
169190
void handle_query(std::size_t bytes);
@@ -188,6 +209,14 @@ class DnsService : public core::IService<DnsService>,
188209
[[nodiscard]] std::vector<uint8_t> build_nxdomain(
189210
const unsigned char* query_data, std::size_t query_len);
190211

212+
/// Build a DNS response with multiple A records (for region-wildcard SEIP queries).
213+
/// Caps at 5 IPs to stay within the 512-byte UDP limit.
214+
[[nodiscard]] std::vector<uint8_t> build_multi_a_response(
215+
const unsigned char* query_data, std::size_t query_len,
216+
const std::string& qname,
217+
const std::vector<std::string>& ips,
218+
uint32_t ttl);
219+
191220
// --- Dynamic record lookup ---
192221
[[nodiscard]] std::optional<std::string> lookup_dynamic_txt(const std::string& fqdn);
193222
[[nodiscard]] std::optional<std::string> lookup_dynamic_a(const std::string& fqdn);
@@ -225,6 +254,11 @@ class DnsService : public core::IService<DnsService>,
225254
// SOA state
226255
std::string soa_email_; // e.g. "admin.domain.com"
227256
std::atomic<uint32_t> soa_serial_{1}; // auto-incremented on zone changes
257+
258+
// SEIP server discovery
259+
std::string server_region_; // this server's region
260+
std::string seip_server_id_; // cached for update_load() re-publish
261+
std::string seip_server_fqdn_; // cached server FQDN for TXT host= field
228262
};
229263

230264
} // namespace nexus::network

projects/LemonadeNexus/src/Core/ServerConfig.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ void to_json(json& j, const ServerConfig& c) {
4141
{"acme_eab_hmac_key", c.acme_eab_hmac_key},
4242
{"dns_provider", c.dns_provider},
4343
{"public_ip", c.public_ip},
44+
{"region", c.region},
4445
{"server_hostname", c.server_hostname},
4546
{"auto_tls", c.auto_tls},
4647
{"dns_base_domain", c.dns_base_domain},
@@ -89,6 +90,7 @@ void from_json(const json& j, ServerConfig& c) {
8990
if (j.contains("acme_eab_hmac_key")) j.at("acme_eab_hmac_key").get_to(c.acme_eab_hmac_key);
9091
if (j.contains("dns_provider")) j.at("dns_provider").get_to(c.dns_provider);
9192
if (j.contains("public_ip")) j.at("public_ip").get_to(c.public_ip);
93+
if (j.contains("region")) j.at("region").get_to(c.region);
9294
if (j.contains("server_hostname")) j.at("server_hostname").get_to(c.server_hostname);
9395
if (j.contains("auto_tls")) j.at("auto_tls").get_to(c.auto_tls);
9496
if (j.contains("dns_base_domain")) j.at("dns_base_domain").get_to(c.dns_base_domain);
@@ -157,6 +159,7 @@ void print_usage(const char* prog) {
157159
spdlog::info(" --tls-cert-path <path> Path to TLS certificate PEM (manual override)");
158160
spdlog::info(" --tls-key-path <path> Path to TLS private key PEM (manual override)");
159161
spdlog::info(" --no-auto-tls Disable automatic TLS certificate via ACME");
162+
spdlog::info(" --region <code> Cloud region (e.g. us-east, eu-west; auto-detected if omitted)");
160163
spdlog::info(" --require-tee Require TEE hardware attestation for Tier 1");
161164
spdlog::info(" --tee-platform <name> Override TEE platform detection (sgx/tdx/sev-snp/secure-enclave)");
162165
spdlog::info(" --help, -h Show this help");
@@ -273,6 +276,8 @@ ServerConfig load_config(int argc, char* argv[]) {
273276
config.require_tee_attestation = true;
274277
} else if (std::strcmp(argv[i], "--tee-platform") == 0 && i + 1 < argc) {
275278
config.tee_platform_override = argv[++i];
279+
} else if (std::strcmp(argv[i], "--region") == 0 && i + 1 < argc) {
280+
config.region = argv[++i];
276281
}
277282
}
278283

@@ -307,6 +312,7 @@ ServerConfig load_config(int argc, char* argv[]) {
307312
if (const char* v = std::getenv("SP_ENROLLMENT_QUORUM")) config.enrollment_quorum_ratio = std::atof(v);
308313
if (std::getenv("SP_REQUIRE_TEE")) config.require_tee_attestation = true;
309314
if (const char* v = std::getenv("SP_TEE_PLATFORM")) config.tee_platform_override = v;
315+
if (const char* v = std::getenv("SP_REGION")) config.region = v;
310316
if (const char* v = std::getenv("SP_SERVER_HOSTNAME")) config.server_hostname = v;
311317
if (const char* v = std::getenv("SP_ACME_EAB_KID")) config.acme_eab_kid = v;
312318
if (const char* v = std::getenv("SP_ACME_EAB_HMAC_KEY")) config.acme_eab_hmac_key = v;

projects/LemonadeNexus/src/Core/ServerIdentity.cpp

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,42 @@ void resolve_server_hostname(
110110
config.server_hostname, region_code);
111111
}
112112

113+
void resolve_server_region(ServerConfig& config,
114+
const std::filesystem::path& data_root) {
115+
if (!config.region.empty()) {
116+
spdlog::info("Region: configured as '{}'", config.region);
117+
return;
118+
}
119+
120+
// Try loading persisted region
121+
auto region_path = data_root / "identity" / "region";
122+
if (std::filesystem::exists(region_path)) {
123+
std::ifstream ifs(region_path);
124+
std::string persisted;
125+
std::getline(ifs, persisted);
126+
if (!persisted.empty()) {
127+
config.region = persisted;
128+
spdlog::info("Region: loaded persisted '{}'", config.region);
129+
return;
130+
}
131+
}
132+
133+
// Auto-detect via HostnameGenerator (HTTP geo-IP lookup)
134+
auto detected = HostnameGenerator::detect_region();
135+
if (detected) {
136+
config.region = *detected;
137+
// Persist for stable region across restarts
138+
std::filesystem::create_directories(region_path.parent_path());
139+
std::ofstream ofs(region_path);
140+
ofs << config.region;
141+
spdlog::info("Region: auto-detected '{}', persisted", config.region);
142+
} else {
143+
config.region = "unknown";
144+
spdlog::warn("Region: auto-detection failed, using 'unknown'. "
145+
"Set --region <code> for accurate geo-aware discovery.");
146+
}
147+
}
148+
113149
std::string resolve_public_ip(const ServerConfig& config) {
114150
std::string ip = config.public_ip;
115151
if (ip.empty() && config.bind_address != "0.0.0.0" && !config.bind_address.empty()) {

0 commit comments

Comments
 (0)