Merge branch 'master' into dev

# Conflicts:
#	mytonctrl/mytonctrl.py
#	mytonctrl/scripts/upgrade.sh

Author: yungwine

modules/validator.py

@@ -1,5 +1,8 @@
-from mypylib.mypylib import color_print
+import time
+
+from mypylib.mypylib import color_print, get_timestamp
 from modules.module import MtcModule
+from mytonctrl.utils import timestamp2utcdatetime, GetColorInt
 
 
 class ValidatorModule(MtcModule):
@@ -32,7 +35,61 @@ def vote_complaint(self, args):
         self.ton.VoteComplaint(election_id, complaint_hash)
         color_print("VoteComplaint - {green}OK{endc}")
 
+    def find_myself(self, validators: list) -> dict:
+        adnl_addr = self.ton.GetAdnlAddr()
+        for validator in validators:
+            if validator.get("adnlAddr") == adnl_addr:
+                return validator
+        return None
+
+    def check_efficiency(self, args):
+        self.local.add_log("start GetValidatorEfficiency function", "debug")
+        previous_validators = self.ton.GetValidatorsList(past=True)
+        validators = self.ton.GetValidatorsList()
+        validator = self.find_myself(previous_validators)
+        config32 = self.ton.GetConfig32()
+        config34 = self.ton.GetConfig34()
+        color_print("{cyan}===[ Validator efficiency ]==={endc}")
+        start_time = timestamp2utcdatetime(config32.startWorkTime)
+        end_time = timestamp2utcdatetime(config32.endWorkTime)
+        color_print(f"Previous round time: {{yellow}}from {start_time} to {end_time}{{endc}}")
+        if validator:
+            if validator.is_masterchain == False:
+                print("Validator index is greater than 100 in the previous round - no efficiency data.")
+            elif validator.get('efficiency') is None:
+                print('Failed to get efficiency for the previous round')
+            else:
+                efficiency = 100 if validator.efficiency > 100 else validator.efficiency
+                color_efficiency = GetColorInt(efficiency, 90, logic="more", ending="%")
+                created = validator.blocks_created
+                expected = validator.blocks_expected
+                color_print(f"Previous round efficiency: {color_efficiency} {{yellow}}({created} blocks created / {round(expected, 1)} blocks expected){{endc}}")
+        else:
+            print("Couldn't find this validator in the previous round")
+        validator = self.find_myself(validators)
+        start_time = timestamp2utcdatetime(config34.startWorkTime)
+        end_time = timestamp2utcdatetime(int(get_timestamp()))
+        color_print(f"Current round time: {{green}}from {start_time} to {end_time}{{endc}}")
+        if validator:
+            if validator.is_masterchain == False:
+                print("Validator index is greater than 100 in the current round - no efficiency data.")
+            elif (time.time() - config34.startWorkTime) / (config34.endWorkTime - config34.startWorkTime) < 0.8:
+                print("The validation round has started recently, there is not enough data yet. "
+                      "The efficiency evaluation will become more accurate towards the end of the round.")
+            elif validator.get('efficiency') is None:
+                print('Failed to get efficiency for the current round')
+            else:
+                efficiency = 100 if validator.efficiency > 100 else validator.efficiency
+                color_efficiency = GetColorInt(efficiency, 90, logic="more", ending="%")
+                created = validator.blocks_created
+                expected = validator.blocks_expected
+                color_print(f"Current round efficiency: {color_efficiency} {{yellow}}({created} blocks created / {round(expected, 1)} blocks expected){{endc}}")
+        else:
+            print("Couldn't find this validator in the current round")
+    # end define
+
     def add_console_commands(self, console):
         console.AddItem("vo", self.vote_offer, self.local.translate("vo_cmd"))
         console.AddItem("ve", self.vote_election_entry, self.local.translate("ve_cmd"))
         console.AddItem("vc", self.vote_complaint, self.local.translate("vc_cmd"))
+        console.AddItem("check_ef", self.check_efficiency, self.local.translate("check_ef_cmd"))

mytoncore/functions.py

@@ -515,6 +515,7 @@ def Slashing(local, ton):
 def save_past_events(local, ton):
     local.try_function(ton.GetElectionEntries)
     local.try_function(ton.GetComplaints)
+    local.try_function(ton.GetValidatorsList, args=[True])  # cache past vl
 
 
 def ScanLiteServers(local, ton):

mytoncore/mytoncore.py

@@ -870,9 +870,10 @@ def GetConfig32(self):
 		#end if
 
 		self.local.add_log("start GetConfig32 function", "debug")
-		config32 = dict()
+		config32 = Dict()
 		result = self.liteClient.Run("getconfig 32")
 		config32["totalValidators"] = int(parse(result, "total:", ' '))
+		config32["mainValidators"] = int(parse(result, "main:", ' '))
 		config32["startWorkTime"] = int(parse(result, "utime_since:", ' '))
 		config32["endWorkTime"] = int(parse(result, "utime_until:", ' '))
 		lines = result.split('\n')
@@ -885,7 +886,7 @@ def GetConfig32(self):
 					validatorWeight = int(parse(line, "weight:", ' '))
 				except ValueError:
 					validatorWeight = int(parse(line, "weight:", ')'))
-				buff = dict()
+				buff = Dict()
 				buff["adnlAddr"] = validatorAdnlAddr
 				buff["pubkey"] = pubkey
 				buff["weight"] = validatorWeight
@@ -906,9 +907,10 @@ def GetConfig34(self):
 		#end if
 
 		self.local.add_log("start GetConfig34 function", "debug")
-		config34 = dict()
+		config34 = Dict()
 		result = self.liteClient.Run("getconfig 34")
 		config34["totalValidators"] = int(parse(result, "total:", ' '))
+		config34["mainValidators"] = int(parse(result, "main:", ' '))
 		config34["startWorkTime"] = int(parse(result, "utime_since:", ' '))
 		config34["endWorkTime"] = int(parse(result, "utime_until:", ' '))
 		config34["totalWeight"] = int(parse(result, "total_weight:", ' '))
@@ -922,7 +924,7 @@ def GetConfig34(self):
 					validatorWeight = int(parse(line, "weight:", ' '))
 				except ValueError:
 					validatorWeight = int(parse(line, "weight:", ')'))
-				buff = dict()
+				buff = Dict()
 				buff["adnlAddr"] = validatorAdnlAddr
 				buff["pubkey"] = pubkey
 				buff["weight"] = validatorWeight
@@ -2152,6 +2154,19 @@ def GetSaveComplaints(self):
 		return saveComplaints
 	#end define
 
+	def GetSaveVl(self):
+		timestamp = get_timestamp()
+		save_vl = self.local.db.get("saveValidatorsLoad")
+		if save_vl is None:
+			save_vl = dict()
+			self.local.db["saveValidatorsLoad"] = save_vl
+		for key, item in list(save_vl.items()):
+			diff_time = timestamp - int(key)
+			if diff_time > 172800:  # 48 hours
+				save_vl.pop(key)
+		return save_vl
+	#end define
+
 	def GetAdnlFromPubkey(self, inputPubkey):
 		config32 = self.GetConfig32()
 		validators = config32["validators"]
@@ -2299,12 +2314,17 @@ def get_valid_complaints(self, complaints: dict, election_id: int):
 				pseudohash = pubkey + str(election_id)
 				if pseudohash == complaint['pseudohash']:
 					exists = True
+					vid = item['id']
 					break
 
 			if not exists:
 				self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint info was not found, probably it's wrong", "info")
 				continue
 
+			if vid >= config32['mainValidators']:
+				self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint created for non masterchain validator", "info")
+				continue
+
 			# check complaint fine value
 			if complaint['suggestedFine'] != 101:  # https://github.com/ton-blockchain/ton/blob/5847897b3758bc9ea85af38e7be8fc867e4c133a/lite-client/lite-client.cpp#L3708
 				self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint fine value is {complaint['suggestedFine']} ton", "info")
@@ -2318,7 +2338,7 @@ def get_valid_complaints(self, complaints: dict, election_id: int):
 
 	def GetOnlineValidators(self):
 		onlineValidators = list()
-		validators = self.GetValidatorsList()
+		validators = self.GetValidatorsList(fast=True)
 		for validator in validators:
 			online = validator.get("online")
 			if online is True:
@@ -2335,7 +2355,6 @@ def GetValidatorsLoad(self, start, end, saveCompFiles=False) -> dict:
 		if buff:
 			return buff
 		#end if
-
 		text = "start GetValidatorsLoad function ({}, {})".format(start, end)
 		self.local.add_log(text, "debug")
 		if saveCompFiles is True:
@@ -2411,7 +2430,7 @@ def GetValidatorsLoad(self, start, end, saveCompFiles=False) -> dict:
 		return data
 	#end define
 
-	def GetValidatorsList(self, past=False):
+	def GetValidatorsList(self, past=False, fast=False):
 		# Get buffer
 		bname = "validatorsList" + str(past)
 		buff = self.GetFunctionBuffer(bname, timeout=60)
@@ -2421,13 +2440,21 @@ def GetValidatorsList(self, past=False):
 
 		timestamp = get_timestamp()
 		end = timestamp - 60
-		start = end - 2000
 		config = self.GetConfig34()
+		if fast:
+			start = end - 1000
+		else:
+			start = config.get("startWorkTime")
 		if past:
 			config = self.GetConfig32()
 			start = config.get("startWorkTime")
 			end = config.get("endWorkTime") - 60
+			save_vl = self.GetSaveVl()
+			start_str = str(start)
+			if start_str in save_vl:
+				return save_vl[start_str]
 		#end if
+
 		validatorsLoad = self.GetValidatorsLoad(start, end)
 		validators = config["validators"]
 		electionId = config.get("startWorkTime")
@@ -2440,12 +2467,22 @@ def GetValidatorsList(self, past=False):
 				validator["wr"] = validatorsLoad[vid]["wr"]
 				validator["efficiency"] = validatorsLoad[vid]["efficiency"]
 				validator["online"] = validatorsLoad[vid]["online"]
+				validator["blocks_created"] = validatorsLoad[vid]["masterBlocksCreated"] + validatorsLoad[vid]["workBlocksCreated"]
+				validator["blocks_expected"] = validatorsLoad[vid]["masterBlocksExpected"] + validatorsLoad[vid]["workBlocksExpected"]
+				validator["is_masterchain"] = False
+				if vid < config["mainValidators"]:
+					validator["is_masterchain"] = True
+				if not validator["is_masterchain"]:
+					validator["efficiency"] = round(validator["wr"] * 100, 2)
 			if saveElectionEntries and adnlAddr in saveElectionEntries:
 				validator["walletAddr"] = saveElectionEntries[adnlAddr]["walletAddr"]
 		#end for
 
 		# Set buffer
 		self.SetFunctionBuffer(bname, validators)
+		if past:
+			save_vl = self.GetSaveVl()
+			save_vl[str(start)] = validators
 		return validators
 	#end define
 
@@ -2457,6 +2494,7 @@ def CheckValidators(self, start, end):
 		data = self.GetValidatorsLoad(start, end, saveCompFiles=True)
 		fullElectorAddr = self.GetFullElectorAddr()
 		wallet = self.GetValidatorWallet(mode="vote")
+		config = self.GetConfig32()
 
 		# Check wallet and balance
 		if wallet is None:
@@ -2474,6 +2512,8 @@ def CheckValidators(self, start, end):
 			pseudohash = pubkey + str(electionId)
 			if pseudohash in valid_complaints:
 				continue
+			if item['id'] >= config['mainValidators']:  # do not create complaints for non-masterchain validators
+				continue
 			# Create complaint
 			fileName = self.remove_proofs_from_complaint(fileName)
 			fileName = self.PrepareComplaint(electionId, fileName)
@@ -3188,7 +3228,7 @@ def ImportCertificate(self, pubkey, fileName):
 
 	def GetValidatorsWalletsList(self):
 		result = list()
-		vl = self.GetValidatorsList()
+		vl = self.GetValidatorsList(fast=True)
 		for item in vl:
 			walletAddr = item["walletAddr"]
 			result.append(walletAddr)

mytonctrl/mytonctrl.py

@@ -42,7 +42,7 @@
 )
 from mytoncore.telemetry import is_host_virtual
 from mytonctrl.migrate import run_migrations
-from mytonctrl.utils import GetItemFromList, timestamp2utcdatetime, fix_git_config, is_hex
+from mytonctrl.utils import GetItemFromList, timestamp2utcdatetime, fix_git_config, is_hex, GetColorInt
 
 import sys, getopt, os
 
@@ -542,7 +542,7 @@ def PrintStatus(local, ton, args):
 
 		if opt != "fast":
 			onlineValidators = ton.GetOnlineValidators()
-			validator_efficiency = ton.GetValidatorEfficiency()
+			# validator_efficiency = ton.GetValidatorEfficiency()
 		if onlineValidators:
 			onlineValidators = len(onlineValidators)
 
@@ -726,7 +726,7 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency,
 	color_print(local.translate("local_status_head"))
 	if ton.using_validator():
 		print(validatorIndex_text)
-		print(validatorEfficiency_text)
+		# print(validatorEfficiency_text)
 	print(adnlAddr_text)
 	print(fullnode_adnl_text)
 	if ton.using_validator():
@@ -747,22 +747,6 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency,
 	print()
 #end define
 
-def GetColorInt(data, border, logic, ending=None):
-	if data is None:
-		result = "n/a"
-	elif logic == "more":
-		if data >= border:
-			result = bcolors.green_text(data, ending)
-		else:
-			result = bcolors.red_text(data, ending)
-	elif logic == "less":
-		if data <= border:
-			result = bcolors.green_text(data, ending)
-		else:
-			result = bcolors.red_text(data, ending)
-	return result
-#end define
-
 def GetColorStatus(input):
 	if input == True:
 		result = bcolors.green_text("working")
@@ -839,6 +823,7 @@ def PrintTimes(local, rootWorkchainEnabledTime_int, startWorkTime, oldStartWorkT
 	print(startNextElectionTime_text)
 #end define
 
+
 def GetColorTime(datetime, timestamp):
 	newTimestamp = get_timestamp()
 	if timestamp > newTimestamp:

mytonctrl/resources/translate.json

@@ -199,6 +199,11 @@
 		"ru": "Запустить установщик модулей TON",
 		"zh_TW": "執行 TON 模組的安裝程序"
 	},
+	"check_ef_cmd": {
+		"en": "Check the efficiency of the validator",
+		"ru": "Проверить эффективность валидатора",
+		"zh_TW": "檢查驗證者的效率"
+	},
 	"ton_status_head": {
 		"en": "{cyan}===[ TON network status ]==={endc}",
 		"ru": "{cyan}===[ Статус сети TON ]==={endc}",

mytonctrl/scripts/upgrade.sh

@@ -48,22 +48,24 @@ else
 	systemctl daemon-reload
 fi
 
-# compile openssl_3
-rm -rf ${bindir}/openssl_3
-git clone https://github.com/openssl/openssl ${bindir}/openssl_3
-cd ${bindir}/openssl_3
-git checkout openssl-3.1.4
-./config
-make build_libs -j$(nproc)
-opensslPath=`pwd`
+if [ ! -d "${bindir}/openssl_3" ]; then
+  git clone https://github.com/openssl/openssl ${bindir}/openssl_3
+  cd ${bindir}/openssl_3
+  git checkout openssl-3.1.4
+  ./config
+  make build_libs -j$(nproc)
+  opensslPath=`pwd`
+else
+  opensslPath=${bindir}/openssl_3
+fi
 
 # Go to work dir
-cd ${srcdir}
-rm -rf ${srcdir}/${repo}
+cd ${srcdir}/${repo}
+ls -A1 | xargs rm -rf
 
 # Update code
 echo "https://github.com/${author}/${repo}.git -> ${branch}"
-git clone --recursive https://github.com/${author}/${repo}.git
+git clone --recursive https://github.com/${author}/${repo}.git .
 cd ${repo} && git checkout ${branch}
 export CC=/usr/bin/clang
 export CXX=/usr/bin/clang++

mytonctrl/utils.py

@@ -1,6 +1,8 @@
 import subprocess
 import time
 
+from mypylib.mypylib import bcolors
+
 
 def timestamp2utcdatetime(timestamp, format="%d.%m.%Y %H:%M:%S"):
     datetime = time.gmtime(timestamp)
@@ -36,4 +38,20 @@ def fix_git_config(git_path: str):
             subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=3)
         else:
             raise Exception(f'Failed to check git status: {err}')
-#end define
+# end define
+
+def GetColorInt(data, border, logic, ending=None):
+    if data is None:
+        result = "n/a"
+    elif logic == "more":
+        if data >= border:
+            result = bcolors.green_text(data, ending)
+        else:
+            result = bcolors.red_text(data, ending)
+    elif logic == "less":
+        if data <= border:
+            result = bcolors.green_text(data, ending)
+        else:
+            result = bcolors.red_text(data, ending)
+    return result
+# end define