Merge pull request #347 from ton-blockchain/efficiency

Efficiency

Author: igroman787

modules/validator.py

@@ -1,5 +1,6 @@
-from mypylib.mypylib import color_print
+from mypylib.mypylib import color_print, get_timestamp
 from modules.module import MtcModule
+from mytonctrl.utils import timestamp2utcdatetime, GetColorInt
 
 
 class ValidatorModule(MtcModule):
@@ -32,7 +33,52 @@ def vote_complaint(self, args):
         self.ton.VoteComplaint(election_id, complaint_hash)
         color_print("VoteComplaint - {green}OK{endc}")
 
+    def find_myself(self, validators: list) -> dict:
+        adnl_addr = self.ton.GetAdnlAddr()
+        for validator in validators:
+            if validator.get("adnlAddr") == adnl_addr:
+                return validator
+        return None
+
+    def check_efficiency(self, args):
+        self.local.add_log("start GetValidatorEfficiency function", "debug")
+        previous_validators = self.ton.GetValidatorsList(past=True)
+        validators = self.ton.GetValidatorsList()
+        validator = self.find_myself(previous_validators)
+        config32 = self.ton.GetConfig32()
+        config34 = self.ton.GetConfig34()
+        color_print("{cyan}===[ Validator efficiency ]==={endc}")
+        if validator:
+            efficiency = 100 if validator.efficiency > 100 else validator.efficiency
+            color_efficiency = GetColorInt(efficiency, 90, logic="more", ending="%")
+            created = validator.blocks_created
+            expected = validator.blocks_expected
+            start_time = timestamp2utcdatetime(config32.startWorkTime)
+            end_time = timestamp2utcdatetime(config32.endWorkTime)
+            color_print(f"Previous round efficiency: {color_efficiency} {{yellow}}({created} blocks created / {expected} blocks expected){{endc}}")
+            color_print(f"Previous round time: {{yellow}}from {start_time} to {end_time}{{endc}}")
+        else:
+            print("Couldn't find this validator in the past round")
+        validator = self.find_myself(validators)
+        if validator:
+            efficiency = 100 if validator.efficiency > 100 else validator.efficiency
+            color_efficiency = GetColorInt(efficiency, 90, logic="more", ending="%")
+            created = validator.blocks_created
+            expected = validator.blocks_expected
+            start_time = timestamp2utcdatetime(config34.startWorkTime)
+            end_time = timestamp2utcdatetime(int(get_timestamp()))
+            if validator.is_masterchain == False and efficiency < 90:
+                print("Your validator index is greater than 100.")
+                print("Efficiency before the validation round is complete may be inaccurate and not displayed.")
+            else:
+                color_print(f"Current round efficiency: {color_efficiency} {{yellow}}({created} blocks created / {expected} blocks expected){{endc}}")
+                color_print(f"Current round time: {{green}}from {start_time} to {end_time}{{endc}}")
+        else:
+            print("Couldn't find this validator in the current round")
+    # end define
+
     def add_console_commands(self, console):
         console.AddItem("vo", self.vote_offer, self.local.translate("vo_cmd"))
         console.AddItem("ve", self.vote_election_entry, self.local.translate("ve_cmd"))
         console.AddItem("vc", self.vote_complaint, self.local.translate("vc_cmd"))
+        console.AddItem("check_ef", self.check_efficiency, self.local.translate("check_ef_cmd"))

mytoncore/functions.py

@@ -521,6 +521,7 @@ def Slashing(local, ton):
 def save_past_events(local, ton):
     local.try_function(ton.GetElectionEntries)
     local.try_function(ton.GetComplaints)
+    local.try_function(ton.GetValidatorsList, args=[True])  # cache past vl
 
 
 def ScanLiteServers(local, ton):

mytoncore/mytoncore.py

@@ -913,9 +913,10 @@ def GetConfig32(self):
 		#end if
 
 		self.local.add_log("start GetConfig32 function", "debug")
-		config32 = dict()
+		config32 = Dict()
 		result = self.liteClient.Run("getconfig 32")
 		config32["totalValidators"] = int(parse(result, "total:", ' '))
+		config32["mainValidators"] = int(parse(result, "main:", ' '))
 		config32["startWorkTime"] = int(parse(result, "utime_since:", ' '))
 		config32["endWorkTime"] = int(parse(result, "utime_until:", ' '))
 		lines = result.split('\n')
@@ -928,7 +929,7 @@ def GetConfig32(self):
 					validatorWeight = int(parse(line, "weight:", ' '))
 				except ValueError:
 					validatorWeight = int(parse(line, "weight:", ')'))
-				buff = dict()
+				buff = Dict()
 				buff["adnlAddr"] = validatorAdnlAddr
 				buff["pubkey"] = pubkey
 				buff["weight"] = validatorWeight
@@ -949,9 +950,10 @@ def GetConfig34(self):
 		#end if
 
 		self.local.add_log("start GetConfig34 function", "debug")
-		config34 = dict()
+		config34 = Dict()
 		result = self.liteClient.Run("getconfig 34")
 		config34["totalValidators"] = int(parse(result, "total:", ' '))
+		config34["mainValidators"] = int(parse(result, "main:", ' '))
 		config34["startWorkTime"] = int(parse(result, "utime_since:", ' '))
 		config34["endWorkTime"] = int(parse(result, "utime_until:", ' '))
 		config34["totalWeight"] = int(parse(result, "total_weight:", ' '))
@@ -965,7 +967,7 @@ def GetConfig34(self):
 					validatorWeight = int(parse(line, "weight:", ' '))
 				except ValueError:
 					validatorWeight = int(parse(line, "weight:", ')'))
-				buff = dict()
+				buff = Dict()
 				buff["adnlAddr"] = validatorAdnlAddr
 				buff["pubkey"] = pubkey
 				buff["weight"] = validatorWeight
@@ -2320,6 +2322,19 @@ def GetSaveComplaints(self):
 		return saveComplaints
 	#end define
 
+	def GetSaveVl(self):
+		timestamp = get_timestamp()
+		save_vl = self.local.db.get("saveValidatorsLoad")
+		if save_vl is None:
+			save_vl = dict()
+			self.local.db["saveValidatorsLoad"] = save_vl
+		for key, item in list(save_vl.items()):
+			diff_time = timestamp - int(key)
+			if diff_time > 172800:  # 48 hours
+				save_vl.pop(key)
+		return save_vl
+	#end define
+
 	def GetAdnlFromPubkey(self, inputPubkey):
 		config32 = self.GetConfig32()
 		validators = config32["validators"]
@@ -2467,12 +2482,17 @@ def get_valid_complaints(self, complaints: dict, election_id: int):
 				pseudohash = pubkey + str(election_id)
 				if pseudohash == complaint['pseudohash']:
 					exists = True
+					vid = item['id']
 					break
 
 			if not exists:
 				self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint info was not found, probably it's wrong", "info")
 				continue
 
+			if vid >= config32['mainValidators']:
+				self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint created for non masterchain validator", "info")
+				continue
+
 			# check complaint fine value
 			if complaint['suggestedFine'] != 101:  # https://github.com/ton-blockchain/ton/blob/5847897b3758bc9ea85af38e7be8fc867e4c133a/lite-client/lite-client.cpp#L3708
 				self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint fine value is {complaint['suggestedFine']} ton", "info")
@@ -2486,7 +2506,7 @@ def get_valid_complaints(self, complaints: dict, election_id: int):
 
 	def GetOnlineValidators(self):
 		onlineValidators = list()
-		validators = self.GetValidatorsList()
+		validators = self.GetValidatorsList(fast=True)
 		for validator in validators:
 			online = validator.get("online")
 			if online is True:
@@ -2503,7 +2523,6 @@ def GetValidatorsLoad(self, start, end, saveCompFiles=False) -> dict:
 		if buff:
 			return buff
 		#end if
-
 		text = "start GetValidatorsLoad function ({}, {})".format(start, end)
 		self.local.add_log(text, "debug")
 		if saveCompFiles is True:
@@ -2579,7 +2598,7 @@ def GetValidatorsLoad(self, start, end, saveCompFiles=False) -> dict:
 		return data
 	#end define
 
-	def GetValidatorsList(self, past=False):
+	def GetValidatorsList(self, past=False, fast=False):
 		# Get buffer
 		bname = "validatorsList" + str(past)
 		buff = self.GetFunctionBuffer(bname, timeout=60)
@@ -2589,13 +2608,21 @@ def GetValidatorsList(self, past=False):
 
 		timestamp = get_timestamp()
 		end = timestamp - 60
-		start = end - 2000
 		config = self.GetConfig34()
+		if fast:
+			start = end - 1000
+		else:
+			start = config.get("startWorkTime")
 		if past:
 			config = self.GetConfig32()
 			start = config.get("startWorkTime")
 			end = config.get("endWorkTime") - 60
+			save_vl = self.GetSaveVl()
+			start_str = str(start)
+			if start_str in save_vl:
+				return save_vl[start_str]
 		#end if
+
 		validatorsLoad = self.GetValidatorsLoad(start, end)
 		validators = config["validators"]
 		electionId = config.get("startWorkTime")
@@ -2608,12 +2635,22 @@ def GetValidatorsList(self, past=False):
 				validator["wr"] = validatorsLoad[vid]["wr"]
 				validator["efficiency"] = validatorsLoad[vid]["efficiency"]
 				validator["online"] = validatorsLoad[vid]["online"]
+				validator["blocks_created"] = validatorsLoad[vid]["masterBlocksCreated"] + validatorsLoad[vid]["workBlocksCreated"]
+				validator["blocks_expected"] = validatorsLoad[vid]["masterBlocksExpected"] + validatorsLoad[vid]["workBlocksExpected"]
+				validator["is_masterchain"] = False
+				if vid < config["mainValidators"]:
+					validator["is_masterchain"] = True
+				if not validator["is_masterchain"]:
+					validator["efficiency"] = round(validator["wr"] * 100, 2)
 			if saveElectionEntries and adnlAddr in saveElectionEntries:
 				validator["walletAddr"] = saveElectionEntries[adnlAddr]["walletAddr"]
 		#end for
 
 		# Set buffer
 		self.SetFunctionBuffer(bname, validators)
+		if past:
+			save_vl = self.GetSaveVl()
+			save_vl[start] = validators
 		return validators
 	#end define
 
@@ -2625,6 +2662,7 @@ def CheckValidators(self, start, end):
 		data = self.GetValidatorsLoad(start, end, saveCompFiles=True)
 		fullElectorAddr = self.GetFullElectorAddr()
 		wallet = self.GetValidatorWallet(mode="vote")
+		config = self.GetConfig32()
 
 		# Check wallet and balance
 		if wallet is None:
@@ -2642,6 +2680,8 @@ def CheckValidators(self, start, end):
 			pseudohash = pubkey + str(electionId)
 			if pseudohash in valid_complaints:
 				continue
+			if item['id'] >= config['mainValidators']:  # do not create complaints for non-masterchain validators
+				continue
 			# Create complaint
 			fileName = self.remove_proofs_from_complaint(fileName)
 			fileName = self.PrepareComplaint(electionId, fileName)
@@ -3463,7 +3503,7 @@ def ImportCertificate(self, pubkey, fileName):
 
 	def GetValidatorsWalletsList(self):
 		result = list()
-		vl = self.GetValidatorsList()
+		vl = self.GetValidatorsList(fast=True)
 		for item in vl:
 			walletAddr = item["walletAddr"]
 			result.append(walletAddr)

mytonctrl/mytonctrl.py

@@ -42,7 +42,7 @@
 )
 from mytoncore.telemetry import is_host_virtual
 from mytonctrl.migrate import run_migrations
-from mytonctrl.utils import GetItemFromList, timestamp2utcdatetime, fix_git_config
+from mytonctrl.utils import GetItemFromList, timestamp2utcdatetime, fix_git_config, GetColorInt
 
 import sys, getopt, os
 
@@ -578,7 +578,7 @@ def PrintStatus(local, ton, args):
 
 		if opt != "fast":
 			onlineValidators = ton.GetOnlineValidators()
-			validator_efficiency = ton.GetValidatorEfficiency()
+			# validator_efficiency = ton.GetValidatorEfficiency()
 		if onlineValidators:
 			onlineValidators = len(onlineValidators)
 
@@ -761,7 +761,7 @@ def PrintLocalStatus(local, adnlAddr, validatorIndex, validatorEfficiency, valid
 
 	color_print(local.translate("local_status_head"))
 	print(validatorIndex_text)
-	print(validatorEfficiency_text)
+	# print(validatorEfficiency_text)
 	print(adnlAddr_text)
 	print(fullnode_adnl_text)
 	print(walletAddr_text)
@@ -781,22 +781,6 @@ def PrintLocalStatus(local, adnlAddr, validatorIndex, validatorEfficiency, valid
 	print()
 #end define
 
-def GetColorInt(data, border, logic, ending=None):
-	if data is None:
-		result = "n/a"
-	elif logic == "more":
-		if data >= border:
-			result = bcolors.green_text(data, ending)
-		else:
-			result = bcolors.red_text(data, ending)
-	elif logic == "less":
-		if data <= border:
-			result = bcolors.green_text(data, ending)
-		else:
-			result = bcolors.red_text(data, ending)
-	return result
-#end define
-
 def GetColorStatus(input):
 	if input == True:
 		result = bcolors.green_text("working")
@@ -873,6 +857,7 @@ def PrintTimes(local, rootWorkchainEnabledTime_int, startWorkTime, oldStartWorkT
 	print(startNextElectionTime_text)
 #end define
 
+
 def GetColorTime(datetime, timestamp):
 	newTimestamp = get_timestamp()
 	if timestamp > newTimestamp:
@@ -1344,7 +1329,8 @@ def PrintElectionEntriesList(ton, args):
 
 def PrintValidatorList(ton, args):
 	past = "past" in args
-	data = ton.GetValidatorsList(past=past)
+	fast = "fast" in args
+	data = ton.GetValidatorsList(past=past, fast=fast)
 	if (data is None or len(data) == 0):
 		print("No data")
 		return
@@ -1353,8 +1339,8 @@ def PrintValidatorList(ton, args):
 		print(text)
 	else:
 		table = list()
-		table += [["ADNL", "Pubkey", "Wallet", "Efficiency", "Online"]]
-		for item in data:
+		table += [["id", "ADNL", "Pubkey", "Wallet", "Efficiency", "Online"]]
+		for i, item in enumerate(data):
 			adnl = item.get("adnlAddr")
 			pubkey = item.get("pubkey")
 			walletAddr = item.get("walletAddr")
@@ -1372,7 +1358,7 @@ def PrintValidatorList(ton, args):
 				online = bcolors.green_text("true")
 			if online == False:
 				online = bcolors.red_text("false")
-			table += [[adnl, pubkey, walletAddr, efficiency, online]]
+			table += [[str(i), adnl, pubkey, walletAddr, efficiency, online]]
 		print_table(table)
 #end define
 

mytonctrl/resources/translate.json

@@ -219,6 +219,11 @@
 		"ru": "Запустить установщик модулей TON",
 		"zh_TW": "執行 TON 模組的安裝程序"
 	},
+	"check_ef_cmd": {
+		"en": "Check the efficiency of the validator",
+		"ru": "Проверить эффективность валидатора",
+		"zh_TW": "檢查驗證者的效率"
+	},
 	"ton_status_head": {
 		"en": "{cyan}===[ TON network status ]==={endc}",
 		"ru": "{cyan}===[ Статус сети TON ]==={endc}",

mytonctrl/utils.py

@@ -1,6 +1,8 @@
 import subprocess
 import time
 
+from mypylib.mypylib import bcolors
+
 
 def timestamp2utcdatetime(timestamp, format="%d.%m.%Y %H:%M:%S"):
     datetime = time.gmtime(timestamp)
@@ -28,4 +30,20 @@ def fix_git_config(git_path: str):
             subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=3)
         else:
             raise Exception(f'Failed to check git status: {err}')
-#end define
+# end define
+
+def GetColorInt(data, border, logic, ending=None):
+    if data is None:
+        result = "n/a"
+    elif logic == "more":
+        if data >= border:
+            result = bcolors.green_text(data, ending)
+        else:
+            result = bcolors.red_text(data, ending)
+    elif logic == "less":
+        if data <= border:
+            result = bcolors.green_text(data, ending)
+        else:
+            result = bcolors.red_text(data, ending)
+    return result
+# end define