Skip to content

Dev #352

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 24 commits into from
Sep 5, 2024
Merged

Dev #352

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions modules/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ class Setting:
'useDefaultCustomOverlays': Setting(None, True, 'Participate in default custom overlays node eligible to'),
'defaultCustomOverlaysUrl': Setting(None, 'https://ton-blockchain.github.io/fallback_custom_overlays.json', 'Default custom overlays config url'),
'debug': Setting(None, False, 'Debug mtc console mode. Prints Traceback on errors'),
'subscribe_tg_channel': Setting('validator', False, 'Disables warning about subscribing to the `TON STATUS` channel'),
}


Expand Down
59 changes: 58 additions & 1 deletion modules/validator.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
from mypylib.mypylib import color_print
import time

from mypylib.mypylib import color_print, get_timestamp
from modules.module import MtcModule
from mytonctrl.utils import timestamp2utcdatetime, GetColorInt


class ValidatorModule(MtcModule):
Expand Down Expand Up @@ -32,7 +35,61 @@ def vote_complaint(self, args):
self.ton.VoteComplaint(election_id, complaint_hash)
color_print("VoteComplaint - {green}OK{endc}")

def find_myself(self, validators: list) -> dict:
adnl_addr = self.ton.GetAdnlAddr()
for validator in validators:
if validator.get("adnlAddr") == adnl_addr:
return validator
return None

def check_efficiency(self, args):
self.local.add_log("start GetValidatorEfficiency function", "debug")
previous_validators = self.ton.GetValidatorsList(past=True)
validators = self.ton.GetValidatorsList()
validator = self.find_myself(previous_validators)
config32 = self.ton.GetConfig32()
config34 = self.ton.GetConfig34()
color_print("{cyan}===[ Validator efficiency ]==={endc}")
start_time = timestamp2utcdatetime(config32.startWorkTime)
end_time = timestamp2utcdatetime(config32.endWorkTime)
color_print(f"Previous round time: {{yellow}}from {start_time} to {end_time}{{endc}}")
if validator:
if validator.is_masterchain == False:
print("Validator index is greater than 100 in the previous round - no efficiency data.")
elif validator.get('efficiency') is None:
print('Failed to get efficiency for the previous round')
else:
efficiency = 100 if validator.efficiency > 100 else validator.efficiency
color_efficiency = GetColorInt(efficiency, 90, logic="more", ending="%")
created = validator.blocks_created
expected = validator.blocks_expected
color_print(f"Previous round efficiency: {color_efficiency} {{yellow}}({created} blocks created / {round(expected, 1)} blocks expected){{endc}}")
else:
print("Couldn't find this validator in the previous round")
validator = self.find_myself(validators)
start_time = timestamp2utcdatetime(config34.startWorkTime)
end_time = timestamp2utcdatetime(int(get_timestamp()))
color_print(f"Current round time: {{green}}from {start_time} to {end_time}{{endc}}")
if validator:
if validator.is_masterchain == False:
print("Validator index is greater than 100 in the current round - no efficiency data.")
elif (time.time() - config34.startWorkTime) / (config34.endWorkTime - config34.startWorkTime) < 0.8:
print("The validation round has started recently, there is not enough data yet. "
"The efficiency evaluation will become more accurate towards the end of the round.")
elif validator.get('efficiency') is None:
print('Failed to get efficiency for the current round')
else:
efficiency = 100 if validator.efficiency > 100 else validator.efficiency
color_efficiency = GetColorInt(efficiency, 90, logic="more", ending="%")
created = validator.blocks_created
expected = validator.blocks_expected
color_print(f"Current round efficiency: {color_efficiency} {{yellow}}({created} blocks created / {round(expected, 1)} blocks expected){{endc}}")
else:
print("Couldn't find this validator in the current round")
# end define

def add_console_commands(self, console):
console.AddItem("vo", self.vote_offer, self.local.translate("vo_cmd"))
console.AddItem("ve", self.vote_election_entry, self.local.translate("ve_cmd"))
console.AddItem("vc", self.vote_complaint, self.local.translate("vc_cmd"))
console.AddItem("check_ef", self.check_efficiency, self.local.translate("check_ef_cmd"))
1 change: 1 addition & 0 deletions mytoncore/functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -515,6 +515,7 @@ def Slashing(local, ton):
def save_past_events(local, ton):
local.try_function(ton.GetElectionEntries)
local.try_function(ton.GetComplaints)
local.try_function(ton.GetValidatorsList, args=[True]) # cache past vl


def ScanLiteServers(local, ton):
Expand Down
58 changes: 49 additions & 9 deletions mytoncore/mytoncore.py
Original file line number Diff line number Diff line change
Expand Up @@ -870,9 +870,10 @@ def GetConfig32(self):
#end if

self.local.add_log("start GetConfig32 function", "debug")
config32 = dict()
config32 = Dict()
result = self.liteClient.Run("getconfig 32")
config32["totalValidators"] = int(parse(result, "total:", ' '))
config32["mainValidators"] = int(parse(result, "main:", ' '))
config32["startWorkTime"] = int(parse(result, "utime_since:", ' '))
config32["endWorkTime"] = int(parse(result, "utime_until:", ' '))
lines = result.split('\n')
Expand All @@ -885,7 +886,7 @@ def GetConfig32(self):
validatorWeight = int(parse(line, "weight:", ' '))
except ValueError:
validatorWeight = int(parse(line, "weight:", ')'))
buff = dict()
buff = Dict()
buff["adnlAddr"] = validatorAdnlAddr
buff["pubkey"] = pubkey
buff["weight"] = validatorWeight
Expand All @@ -906,9 +907,10 @@ def GetConfig34(self):
#end if

self.local.add_log("start GetConfig34 function", "debug")
config34 = dict()
config34 = Dict()
result = self.liteClient.Run("getconfig 34")
config34["totalValidators"] = int(parse(result, "total:", ' '))
config34["mainValidators"] = int(parse(result, "main:", ' '))
config34["startWorkTime"] = int(parse(result, "utime_since:", ' '))
config34["endWorkTime"] = int(parse(result, "utime_until:", ' '))
config34["totalWeight"] = int(parse(result, "total_weight:", ' '))
Expand All @@ -922,7 +924,7 @@ def GetConfig34(self):
validatorWeight = int(parse(line, "weight:", ' '))
except ValueError:
validatorWeight = int(parse(line, "weight:", ')'))
buff = dict()
buff = Dict()
buff["adnlAddr"] = validatorAdnlAddr
buff["pubkey"] = pubkey
buff["weight"] = validatorWeight
Expand Down Expand Up @@ -2152,6 +2154,19 @@ def GetSaveComplaints(self):
return saveComplaints
#end define

def GetSaveVl(self):
timestamp = get_timestamp()
save_vl = self.local.db.get("saveValidatorsLoad")
if save_vl is None:
save_vl = dict()
self.local.db["saveValidatorsLoad"] = save_vl
for key, item in list(save_vl.items()):
diff_time = timestamp - int(key)
if diff_time > 172800: # 48 hours
save_vl.pop(key)
return save_vl
#end define

def GetAdnlFromPubkey(self, inputPubkey):
config32 = self.GetConfig32()
validators = config32["validators"]
Expand Down Expand Up @@ -2299,12 +2314,17 @@ def get_valid_complaints(self, complaints: dict, election_id: int):
pseudohash = pubkey + str(election_id)
if pseudohash == complaint['pseudohash']:
exists = True
vid = item['id']
break

if not exists:
self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint info was not found, probably it's wrong", "info")
continue

if vid >= config32['mainValidators']:
self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint created for non masterchain validator", "info")
continue

# check complaint fine value
if complaint['suggestedFine'] != 101: # https://github.com/ton-blockchain/ton/blob/5847897b3758bc9ea85af38e7be8fc867e4c133a/lite-client/lite-client.cpp#L3708
self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint fine value is {complaint['suggestedFine']} ton", "info")
Expand All @@ -2318,7 +2338,7 @@ def get_valid_complaints(self, complaints: dict, election_id: int):

def GetOnlineValidators(self):
onlineValidators = list()
validators = self.GetValidatorsList()
validators = self.GetValidatorsList(fast=True)
for validator in validators:
online = validator.get("online")
if online is True:
Expand All @@ -2335,7 +2355,6 @@ def GetValidatorsLoad(self, start, end, saveCompFiles=False) -> dict:
if buff:
return buff
#end if

text = "start GetValidatorsLoad function ({}, {})".format(start, end)
self.local.add_log(text, "debug")
if saveCompFiles is True:
Expand Down Expand Up @@ -2411,7 +2430,7 @@ def GetValidatorsLoad(self, start, end, saveCompFiles=False) -> dict:
return data
#end define

def GetValidatorsList(self, past=False):
def GetValidatorsList(self, past=False, fast=False):
# Get buffer
bname = "validatorsList" + str(past)
buff = self.GetFunctionBuffer(bname, timeout=60)
Expand All @@ -2421,13 +2440,21 @@ def GetValidatorsList(self, past=False):

timestamp = get_timestamp()
end = timestamp - 60
start = end - 2000
config = self.GetConfig34()
if fast:
start = end - 1000
else:
start = config.get("startWorkTime")
if past:
config = self.GetConfig32()
start = config.get("startWorkTime")
end = config.get("endWorkTime") - 60
save_vl = self.GetSaveVl()
start_str = str(start)
if start_str in save_vl:
return save_vl[start_str]
#end if

validatorsLoad = self.GetValidatorsLoad(start, end)
validators = config["validators"]
electionId = config.get("startWorkTime")
Expand All @@ -2440,12 +2467,22 @@ def GetValidatorsList(self, past=False):
validator["wr"] = validatorsLoad[vid]["wr"]
validator["efficiency"] = validatorsLoad[vid]["efficiency"]
validator["online"] = validatorsLoad[vid]["online"]
validator["blocks_created"] = validatorsLoad[vid]["masterBlocksCreated"] + validatorsLoad[vid]["workBlocksCreated"]
validator["blocks_expected"] = validatorsLoad[vid]["masterBlocksExpected"] + validatorsLoad[vid]["workBlocksExpected"]
validator["is_masterchain"] = False
if vid < config["mainValidators"]:
validator["is_masterchain"] = True
if not validator["is_masterchain"]:
validator["efficiency"] = round(validator["wr"] * 100, 2)
if saveElectionEntries and adnlAddr in saveElectionEntries:
validator["walletAddr"] = saveElectionEntries[adnlAddr]["walletAddr"]
#end for

# Set buffer
self.SetFunctionBuffer(bname, validators)
if past:
save_vl = self.GetSaveVl()
save_vl[str(start)] = validators
return validators
#end define

Expand All @@ -2457,6 +2494,7 @@ def CheckValidators(self, start, end):
data = self.GetValidatorsLoad(start, end, saveCompFiles=True)
fullElectorAddr = self.GetFullElectorAddr()
wallet = self.GetValidatorWallet(mode="vote")
config = self.GetConfig32()

# Check wallet and balance
if wallet is None:
Expand All @@ -2474,6 +2512,8 @@ def CheckValidators(self, start, end):
pseudohash = pubkey + str(electionId)
if pseudohash in valid_complaints:
continue
if item['id'] >= config['mainValidators']: # do not create complaints for non-masterchain validators
continue
# Create complaint
fileName = self.remove_proofs_from_complaint(fileName)
fileName = self.PrepareComplaint(electionId, fileName)
Expand Down Expand Up @@ -3188,7 +3228,7 @@ def ImportCertificate(self, pubkey, fileName):

def GetValidatorsWalletsList(self):
result = list()
vl = self.GetValidatorsList()
vl = self.GetValidatorsList(fast=True)
for item in vl:
walletAddr = item["walletAddr"]
result.append(walletAddr)
Expand Down
49 changes: 26 additions & 23 deletions mytonctrl/mytonctrl.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
)
from mytoncore.telemetry import is_host_virtual
from mytonctrl.migrate import run_migrations
from mytonctrl.utils import GetItemFromList, timestamp2utcdatetime, fix_git_config, is_hex
from mytonctrl.utils import GetItemFromList, timestamp2utcdatetime, fix_git_config, is_hex, GetColorInt

import sys, getopt, os

Expand Down Expand Up @@ -467,11 +467,29 @@ def check_vps(local, ton):
color_print(f"Virtualization detected: {data['product_name']}")
#end define

def check_tg_channel(local, ton):
if ton.using_validator() and ton.local.db.get("subscribe_tg_channel") is None:
print_warning(local, "subscribe_tg_channel_warning")
#end difine

def check_slashed(local, ton):
config32 = ton.GetConfig32()
save_complaints = ton.GetSaveComplaints()
complaints = save_complaints.get(str(config32['startWorkTime']))
if not complaints:
return
for c in complaints.values():
if c["adnl"] == ton.GetAdnlAddr() and c["isPassed"]:
print_warning(local, "slashed_warning")
#end define

def warnings(local, ton):
check_disk_usage(local, ton)
check_sync(local, ton)
check_validator_balance(local, ton)
check_vps(local, ton)
local.try_function(check_disk_usage, args=[local, ton])
local.try_function(check_sync, args=[local, ton])
local.try_function(check_validator_balance, args=[local, ton])
local.try_function(check_vps, args=[local, ton])
local.try_function(check_tg_channel, args=[local, ton])
local.try_function(check_slashed, args=[local, ton])
#end define

def CheckTonUpdate(local):
Expand Down Expand Up @@ -542,7 +560,7 @@ def PrintStatus(local, ton, args):

if opt != "fast":
onlineValidators = ton.GetOnlineValidators()
validator_efficiency = ton.GetValidatorEfficiency()
# validator_efficiency = ton.GetValidatorEfficiency()
if onlineValidators:
onlineValidators = len(onlineValidators)

Expand Down Expand Up @@ -726,7 +744,7 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency,
color_print(local.translate("local_status_head"))
if ton.using_validator():
print(validatorIndex_text)
print(validatorEfficiency_text)
# print(validatorEfficiency_text)
print(adnlAddr_text)
print(fullnode_adnl_text)
if ton.using_validator():
Expand All @@ -747,22 +765,6 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency,
print()
#end define

def GetColorInt(data, border, logic, ending=None):
if data is None:
result = "n/a"
elif logic == "more":
if data >= border:
result = bcolors.green_text(data, ending)
else:
result = bcolors.red_text(data, ending)
elif logic == "less":
if data <= border:
result = bcolors.green_text(data, ending)
else:
result = bcolors.red_text(data, ending)
return result
#end define

def GetColorStatus(input):
if input == True:
result = bcolors.green_text("working")
Expand Down Expand Up @@ -839,6 +841,7 @@ def PrintTimes(local, rootWorkchainEnabledTime_int, startWorkTime, oldStartWorkT
print(startNextElectionTime_text)
#end define


def GetColorTime(datetime, timestamp):
newTimestamp = get_timestamp()
if timestamp > newTimestamp:
Expand Down
Loading