Improve overlay stats (#1242)

Author: SpyCheese

overlay/overlay-manager.cpp

@@ -188,7 +188,7 @@ void OverlayManager::receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeId
     return;
   }
 
-  td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), false);
+  td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, data.size(), false, false);
   td::actor::send_closure(it2->second.overlay, &Overlay::receive_message, src, std::move(extra), std::move(data));
 }
 
@@ -225,7 +225,14 @@ void OverlayManager::receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdSh
     return;
   }
 
-  td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), true);
+  td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, data.size(), true, false);
+  promise = [overlay = it2->second.overlay.get(), promise = std::move(promise),
+             src](td::Result<td::BufferSlice> R) mutable {
+    if (R.is_ok()) {
+      td::actor::send_closure(overlay, &Overlay::update_throughput_out_ctr, src, R.ok().size(), false, true);
+    }
+    promise.set_result(std::move(R));
+  };
   td::actor::send_closure(it2->second.overlay, &Overlay::receive_query, src, std::move(extra), std::move(data),
                           std::move(promise));
 }
@@ -243,8 +250,14 @@ void OverlayManager::send_query_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdS
   if (it != overlays_.end()) {
     auto it2 = it->second.find(overlay_id);
     if (it2 != it->second.end()) {
-      td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, (td::uint32)query.size(),
-                              true);
+      td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, query.size(), true, false);
+      promise = [overlay = it2->second.overlay.get(), promise = std::move(promise),
+                 dst](td::Result<td::BufferSlice> R) mutable {
+        if (R.is_ok()) {
+          td::actor::send_closure(overlay, &Overlay::update_throughput_in_ctr, dst, R.ok().size(), false, true);
+        }
+        promise.set_result(std::move(R));
+      };
       if (!it2->second.member_certificate.empty()) {
         extra->flags_ |= 1;
         extra->certificate_ = it2->second.member_certificate.tl();
@@ -273,7 +286,7 @@ void OverlayManager::send_message_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeI
   if (it != overlays_.end()) {
     auto it2 = it->second.find(overlay_id);
     if (it2 != it->second.end()) {
-      td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, (td::uint32)object.size(),
+      td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, object.size(), false,
                               false);
       if (!it2->second.member_certificate.empty()) {
         // do not send certificate here, we hope that all our neighbours already know of out certificate

overlay/overlay-peers.cpp

@@ -574,28 +574,40 @@ void OverlayImpl::update_peer_err_ctr(adnl::AdnlNodeIdShort peer_id, bool is_fec
   }
 }
 
-void OverlayImpl::update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) {
+void OverlayImpl::update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query,
+                                            bool is_response) {
   auto out_peer = peer_list_.peers_.get(peer_id);
   if (out_peer) {
-    out_peer->throughput_out_bytes_ctr += msg_size;
-    out_peer->throughput_out_packets_ctr++;
-
+    out_peer->traffic_ctr.add_packet(msg_size, false);
+    if (is_response) {
+      out_peer->traffic_responses_ctr.add_packet(msg_size, false);
+    }
     if (is_query) {
       out_peer->last_out_query_at = td::Timestamp::now();
     }
   }
+  total_traffic_ctr.add_packet(msg_size, false);
+  if (is_response) {
+    total_traffic_responses_ctr.add_packet(msg_size, false);
+  }
 }
 
-void OverlayImpl::update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) {
+void OverlayImpl::update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query,
+                                           bool is_response) {
   auto in_peer = peer_list_.peers_.get(peer_id);
   if (in_peer) {
-    in_peer->throughput_in_bytes_ctr += msg_size;
-    in_peer->throughput_in_packets_ctr++;
-
+    in_peer->traffic_ctr.add_packet(msg_size, true);
+    if (is_response) {
+      in_peer->traffic_responses_ctr.add_packet(msg_size, true);
+    }
     if (is_query) {
       in_peer->last_in_query_at = td::Timestamp::now();
     }
   }
+  total_traffic_ctr.add_packet(msg_size, true);
+  if (is_response) {
+    total_traffic_responses_ctr.add_packet(msg_size, true);
+  }
 }
 
 void OverlayImpl::update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) {

overlay/overlay.cpp

@@ -38,30 +38,32 @@ namespace overlay {
 
 const OverlayMemberCertificate OverlayNode::empty_certificate_{};
 
+static std::string overlay_actor_name(const OverlayIdFull &overlay_id) {
+  return PSTRING() << "overlay." << overlay_id.compute_short_id().bits256_value().to_hex().substr(0, 4);
+}
+
 td::actor::ActorOwn<Overlay> Overlay::create_public(td::actor::ActorId<keyring::Keyring> keyring,
                                                     td::actor::ActorId<adnl::Adnl> adnl,
                                                     td::actor::ActorId<OverlayManager> manager,
                                                     td::actor::ActorId<dht::Dht> dht_node,
                                                     adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
                                                     std::unique_ptr<Overlays::Callback> callback,
                                                     OverlayPrivacyRules rules, td::string scope, OverlayOptions opts) {
-  auto R = td::actor::create_actor<OverlayImpl>(
-      "overlay", keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), OverlayType::Public,
-      std::vector<adnl::AdnlNodeIdShort>(), std::vector<PublicKeyHash>(), OverlayMemberCertificate{}, std::move(callback),
-      std::move(rules), std::move(scope), std::move(opts));
-  return td::actor::ActorOwn<Overlay>(std::move(R));
+  return td::actor::create_actor<OverlayImpl>(
+      overlay_actor_name(overlay_id), keyring, adnl, manager, dht_node, local_id, std::move(overlay_id),
+      OverlayType::Public, std::vector<adnl::AdnlNodeIdShort>(), std::vector<PublicKeyHash>(),
+      OverlayMemberCertificate{}, std::move(callback), std::move(rules), std::move(scope), std::move(opts));
 }
 
 td::actor::ActorOwn<Overlay> Overlay::create_private(
     td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
     td::actor::ActorId<OverlayManager> manager, td::actor::ActorId<dht::Dht> dht_node, adnl::AdnlNodeIdShort local_id,
     OverlayIdFull overlay_id, std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Overlays::Callback> callback,
     OverlayPrivacyRules rules, std::string scope, OverlayOptions opts) {
-  auto R = td::actor::create_actor<OverlayImpl>("overlay", keyring, adnl, manager, dht_node, local_id,
-                                                std::move(overlay_id), OverlayType::FixedMemberList, std::move(nodes),
-                                                std::vector<PublicKeyHash>(), OverlayMemberCertificate{},
-                                                std::move(callback), std::move(rules), std::move(scope));
-  return td::actor::ActorOwn<Overlay>(std::move(R));
+  return td::actor::create_actor<OverlayImpl>(
+      overlay_actor_name(overlay_id), keyring, adnl, manager, dht_node, local_id, std::move(overlay_id),
+      OverlayType::FixedMemberList, std::move(nodes), std::vector<PublicKeyHash>(), OverlayMemberCertificate{},
+      std::move(callback), std::move(rules), std::move(scope));
 }
 
 td::actor::ActorOwn<Overlay> Overlay::create_semiprivate(
@@ -70,11 +72,10 @@ td::actor::ActorOwn<Overlay> Overlay::create_semiprivate(
     OverlayIdFull overlay_id, std::vector<adnl::AdnlNodeIdShort> nodes, std::vector<PublicKeyHash> root_public_keys,
     OverlayMemberCertificate cert, std::unique_ptr<Overlays::Callback> callback, OverlayPrivacyRules rules,
     std::string scope, OverlayOptions opts) {
-  auto R = td::actor::create_actor<OverlayImpl>(
-      "overlay", keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), OverlayType::CertificatedMembers,
-      std::move(nodes), std::move(root_public_keys), std::move(cert), std::move(callback), std::move(rules),
-      std::move(scope), std::move(opts));
-  return td::actor::ActorOwn<Overlay>(std::move(R));
+  return td::actor::create_actor<OverlayImpl>(overlay_actor_name(overlay_id), keyring, adnl, manager, dht_node,
+                                              local_id, std::move(overlay_id), OverlayType::CertificatedMembers,
+                                              std::move(nodes), std::move(root_public_keys), std::move(cert),
+                                              std::move(callback), std::move(rules), std::move(scope), std::move(opts));
 }
 
 OverlayImpl::OverlayImpl(td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
@@ -281,17 +282,12 @@ void OverlayImpl::alarm() {
 
     auto SelfId = actor_id(this);
     iterate_all_peers([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) {
-      peer.throughput_out_bytes = static_cast<td::uint32>(peer.throughput_out_bytes_ctr / t_elapsed);
-      peer.throughput_in_bytes = static_cast<td::uint32>(peer.throughput_in_bytes_ctr / t_elapsed);
-
-      peer.throughput_out_packets = static_cast<td::uint32>(peer.throughput_out_packets_ctr / t_elapsed);
-      peer.throughput_in_packets = static_cast<td::uint32>(peer.throughput_in_packets_ctr / t_elapsed);
-
-      peer.throughput_out_bytes_ctr = 0;
-      peer.throughput_in_bytes_ctr = 0;
-
-      peer.throughput_out_packets_ctr = 0;
-      peer.throughput_in_packets_ctr = 0;
+      peer.traffic = peer.traffic_ctr;
+      peer.traffic.normalize(t_elapsed);
+      peer.traffic_ctr = {};
+      peer.traffic_responses = peer.traffic_responses_ctr;
+      peer.traffic_responses.normalize(t_elapsed);
+      peer.traffic_responses_ctr = {};
 
       auto P = td::PromiseCreator::lambda([SelfId, peer_id = key](td::Result<td::string> result) {
         result.ensure();
@@ -300,6 +296,12 @@ void OverlayImpl::alarm() {
 
       td::actor::send_closure(adnl_, &adnl::AdnlSenderInterface::get_conn_ip_str, local_id_, key, std::move(P));
     });
+    total_traffic = total_traffic_ctr;
+    total_traffic.normalize(t_elapsed);
+    total_traffic_ctr = {};
+    total_traffic_responses = total_traffic_responses_ctr;
+    total_traffic_responses.normalize(t_elapsed);
+    total_traffic_responses_ctr = {};
 
     update_throughput_at_ = td::Timestamp::in(50.0);
     last_throughput_update_ = td::Timestamp::now();
@@ -691,12 +693,8 @@ void OverlayImpl::get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_
   iterate_all_peers([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) {
     auto node_obj = create_tl_object<ton_api::engine_validator_overlayStatsNode>();
     node_obj->adnl_id_ = key.bits256_value();
-    node_obj->t_out_bytes_ = peer.throughput_out_bytes;
-    node_obj->t_in_bytes_ = peer.throughput_in_bytes;
-
-    node_obj->t_out_pckts_ = peer.throughput_out_packets;
-    node_obj->t_in_pckts_ = peer.throughput_in_packets;
-
+    node_obj->traffic_ = peer.traffic.tl();
+    node_obj->traffic_responses_ = peer.traffic_responses.tl();
     node_obj->ip_addr_ = peer.ip_addr_str;
 
     node_obj->last_in_query_ = static_cast<td::uint32>(peer.last_in_query_at.at_unix());
@@ -712,6 +710,8 @@ void OverlayImpl::get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_
     res->nodes_.push_back(std::move(node_obj));
   });
 
+  res->total_traffic_ = total_traffic.tl();
+  res->total_traffic_responses_ = total_traffic_responses.tl();
   res->stats_.push_back(
       create_tl_object<ton_api::engine_validator_oneStat>("neighbours_cnt", PSTRING() << neighbours_cnt()));
 
@@ -732,6 +732,27 @@ bool OverlayImpl::has_valid_broadcast_certificate(const PublicKeyHash &source, s
          BroadcastCheckResult::Forbidden;
 }
 
+void TrafficStats::add_packet(td::uint64 size, bool in) {
+  if (in) {
+    ++in_packets;
+    in_bytes += size;
+  } else {
+    ++out_packets;
+    out_bytes += size;
+  }
+}
+
+void TrafficStats::normalize(double elapsed) {
+  out_bytes = static_cast<td::uint64>(out_bytes / elapsed);
+  in_bytes = static_cast<td::uint64>(in_bytes / elapsed);
+  out_packets = static_cast<td::uint32>(out_packets / elapsed);
+  in_packets = static_cast<td::uint32>(in_packets / elapsed);
+}
+
+tl_object_ptr<ton_api::engine_validator_overlayStatsTraffic> TrafficStats::tl() const {
+  return create_tl_object<ton_api::engine_validator_overlayStatsTraffic>(out_bytes, in_bytes, out_packets, in_packets);
+}
+
 }  // namespace overlay
 
 }  // namespace ton

overlay/overlay.h

@@ -72,8 +72,10 @@ class Overlay : public td::actor::Actor {
   virtual void receive_nodes_from_db(tl_object_ptr<ton_api::overlay_nodes> nodes) = 0;
   virtual void receive_nodes_from_db_v2(tl_object_ptr<ton_api::overlay_nodesV2> nodes) = 0;
   virtual void get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_overlayStats>> promise) = 0;
-  virtual void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) = 0;
-  virtual void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) = 0;
+  virtual void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query,
+                                         bool is_response) = 0;
+  virtual void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query,
+                                        bool is_response) = 0;
   virtual void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) = 0;
   virtual void update_member_certificate(OverlayMemberCertificate cert) = 0;
   virtual void update_root_member_list(std::vector<adnl::AdnlNodeIdShort> nodes,

overlay/overlay.hpp

@@ -59,6 +59,18 @@ namespace overlay {
 
 class OverlayImpl;
 
+struct TrafficStats {
+  td::uint64 out_bytes = 0;
+  td::uint64 in_bytes = 0;
+
+  td::uint32 out_packets = 0;
+  td::uint32 in_packets = 0;
+
+  void add_packet(td::uint64 size, bool in);
+  void normalize(double elapsed);
+  tl_object_ptr<ton_api::engine_validator_overlayStatsTraffic> tl() const;
+};
+
 class OverlayPeer {
  public:
   adnl::AdnlNodeIdShort get_id() const {
@@ -126,17 +138,8 @@ class OverlayPeer {
     return node_.has_full_id();
   }
 
-  td::uint32 throughput_out_bytes = 0;
-  td::uint32 throughput_in_bytes = 0;
-
-  td::uint32 throughput_out_packets = 0;
-  td::uint32 throughput_in_packets = 0;
-
-  td::uint32 throughput_out_bytes_ctr = 0;
-  td::uint32 throughput_in_bytes_ctr = 0;
-
-  td::uint32 throughput_out_packets_ctr = 0;
-  td::uint32 throughput_in_packets_ctr = 0;
+  TrafficStats traffic, traffic_ctr;
+  TrafficStats traffic_responses, traffic_responses_ctr;
 
   td::uint32 broadcast_errors = 0;
   td::uint32 fec_broadcast_errors = 0;
@@ -269,9 +272,11 @@ class OverlayImpl : public Overlay {
 
   void get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_overlayStats>> promise) override;
 
-  void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override;
+  void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query,
+                                 bool is_response) override;
 
-  void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override;
+  void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query,
+                                bool is_response) override;
 
   void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) override;
 
@@ -457,6 +462,8 @@ class OverlayImpl : public Overlay {
     td::Timestamp local_cert_is_valid_until_;
     td::uint32 local_member_flags_{0};
   } peer_list_;
+  TrafficStats total_traffic, total_traffic_ctr;
+  TrafficStats total_traffic_responses, total_traffic_responses_ctr;
 
   OverlayOptions opts_;
 };

tl/generate/scheme/ton_api.tl

@@ -683,10 +683,16 @@ engine.validator.proposalVote perm_key:int256 to_send:bytes = engine.validator.P
 engine.validator.dhtServerStatus id:int256 status:int = engine.validator.DhtServerStatus;
 engine.validator.dhtServersStatus servers:(vector engine.validator.dhtServerStatus) = engine.validator.DhtServersStatus;
 
+engine.validator.overlayStatsTraffic t_out_bytes:long t_in_bytes:long t_out_pckts:int t_in_pckts:int = engine.validator.OverlayStatsTraffic;
+
 engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string is_neighbour:Bool is_alive:Bool node_flags:int
-    bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int t_out_bytes:int t_in_bytes:int t_out_pckts:int t_in_pckts:int = engine.validator.OverlayStatsNode;
+    bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int
+    traffic:engine.validator.overlayStatsTraffic traffic_responses:engine.validator.overlayStatsTraffic = engine.validator.OverlayStatsNode;
 
-engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) extra:string = engine.validator.OverlayStats;
+engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string
+    nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat)
+    total_traffic:engine.validator.overlayStatsTraffic total_traffic_responses:engine.validator.overlayStatsTraffic
+    extra:string = engine.validator.OverlayStats;
 engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats;
 
 engine.validator.onePerfTimerStat time:int min:double avg:double max:double = engine.validator.OnePerfTimerStat;