From dadbbd951a6902a4a0742eebff143405b9ae0e0d Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 3 Jul 2019 00:03:55 +0700 Subject: [PATCH 01/14] AnnounceShare: pass the tracker type to getTrackers() in onTrackerFound() --- plugins/AnnounceShare/AnnounceSharePlugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/AnnounceShare/AnnounceSharePlugin.py b/plugins/AnnounceShare/AnnounceSharePlugin.py index 057ce55a1..d5428f87d 100644 --- a/plugins/AnnounceShare/AnnounceSharePlugin.py +++ b/plugins/AnnounceShare/AnnounceSharePlugin.py @@ -26,7 +26,7 @@ def onTrackerFound(self, tracker_address, type="shared", my=False): if not tracker_address.startswith("zero://"): return False - trackers = self.getTrackers() + trackers = self.getTrackers(type) added = False if tracker_address not in trackers: trackers[tracker_address] = { From fe32f1f9e7aa7dc4eadf3c0fd59973417fcc4584 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 3 Jul 2019 09:48:07 +0700 Subject: [PATCH 02/14] AnnounceShare: allow trackers of any supported protocols, not just zero:// --- plugins/AnnounceShare/AnnounceSharePlugin.py | 36 ++++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/plugins/AnnounceShare/AnnounceSharePlugin.py b/plugins/AnnounceShare/AnnounceSharePlugin.py index d5428f87d..9ee780d20 100644 --- a/plugins/AnnounceShare/AnnounceSharePlugin.py +++ b/plugins/AnnounceShare/AnnounceSharePlugin.py @@ -13,17 +13,46 @@ class TrackerStorage(object): def __init__(self): + self.site_announcer = None self.log = logging.getLogger("TrackerStorage") self.file_path = "%s/trackers.json" % config.data_dir self.load() self.time_discover = 0.0 atexit.register(self.save) + def setSiteAnnouncer(self, site_announcer): + if self.site_announcer == site_announcer: + return + self.site_announcer = site_announcer + self.recheckValidTrackers() + + def isTrackerAddressValid(self, tracker_address): + if not self.site_announcer: # Not completely initialized, skip check + return True + + address_parts = self.site_announcer.getAddressParts(tracker_address) + if not address_parts: + self.log.debug("Invalid tracker address: %s" % tracker_address) + return False + + handler = self.site_announcer.getTrackerHandler(address_parts["protocol"]) + if not handler: + self.log.debug("Invalid tracker address: Unknown protocol %s: %s" % (address_parts["protocol"], tracker_address)) + return False + + return True + + def recheckValidTrackers(self): + trackers = self.getTrackers() + for address, tracker in list(trackers.items()): + if not self.isTrackerAddressValid(address): + del trackers[address] + def getDefaultFile(self): return {"shared": {}} def onTrackerFound(self, tracker_address, type="shared", my=False): - if not tracker_address.startswith("zero://"): + if not self.isTrackerAddressValid(tracker_address): return False trackers = self.getTrackers(type) @@ -97,8 +126,7 @@ def load(self): self.log.debug("Loaded %s shared trackers" % len(trackers)) for address, tracker in list(trackers.items()): tracker["num_error"] = 0 - if not address.startswith("zero://"): - del trackers[address] + self.recheckValidTrackers() def save(self): s = time.time() @@ -142,6 +170,7 @@ def discoverTrackers(self, peers): @PluginManager.registerTo("SiteAnnouncer") class SiteAnnouncerPlugin(object): def getTrackers(self): + tracker_storage.setSiteAnnouncer(self) if tracker_storage.time_discover < time.time() - 5 * 60: tracker_storage.time_discover = time.time() gevent.spawn(tracker_storage.discoverTrackers, self.site.getConnectedPeers()) @@ -153,6 +182,7 @@ def getTrackers(self): return trackers def announceTracker(self, tracker, *args, **kwargs): + tracker_storage.setSiteAnnouncer(self) res = super(SiteAnnouncerPlugin, self).announceTracker(tracker, *args, **kwargs) if res: latency = res From 28eabcd8f38f7f87d58a4679b4210d986889c0e4 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 3 Jul 2019 13:37:03 +0700 Subject: [PATCH 03/14] AnnounceShare: Implement counting of working trackers on per-protocol basis. AnnounceShare tries to autodetect supported tracker protocols and keep the actual list of working trackers for each protocol. The udp:// and http(s):// versions of the BitTorrent protocol are considered as 2 distinct protocols. http:// and https:// are considered as the same protocol. Added option --working_shared_trackers_limit_per_protocol to adjust the minimum number of working trackers for each protocol (3, by default). --- plugins/AnnounceShare/AnnounceSharePlugin.py | 93 +++++++++++++++++++- 1 file changed, 90 insertions(+), 3 deletions(-) diff --git a/plugins/AnnounceShare/AnnounceSharePlugin.py b/plugins/AnnounceShare/AnnounceSharePlugin.py index 9ee780d20..a1c9faabc 100644 --- a/plugins/AnnounceShare/AnnounceSharePlugin.py +++ b/plugins/AnnounceShare/AnnounceSharePlugin.py @@ -48,6 +48,39 @@ def recheckValidTrackers(self): if not self.isTrackerAddressValid(address): del trackers[address] + def getNormalizedTrackerProtocol(self, tracker_address): + if not self.site_announcer: + return None + + address_parts = self.site_announcer.getAddressParts(tracker_address) + if not address_parts: + return None + + protocol = address_parts["protocol"] + if protocol == "https": + protocol == "http" + + return protocol + + def getSupportedProtocols(self): + if not self.site_announcer: + return None + + supported_trackers = self.site_announcer.getSupportedTrackers() + + protocols = {} + for tracker_address in supported_trackers: + protocol = self.getNormalizedTrackerProtocol(tracker_address) + if not protocol: + continue + protocols[protocol] = True + + protocols = list(protocols.keys()) + + self.log.debug("Supported tracker protocols: %s" % protocols) + + return protocols + def getDefaultFile(self): return {"shared": {}} @@ -99,13 +132,40 @@ def onTrackerError(self, tracker_address): self.log.debug("Tracker %s looks down, removing." % tracker_address) del trackers[tracker_address] + def isTrackerWorking(self, tracker_address, type="shared"): + trackers = self.getTrackers(type) + tracker = trackers[tracker_address] + if not tracker: + return False + + if tracker["time_success"] > time.time() - 60 * 60: + return True + + return False + def getTrackers(self, type="shared"): return self.file_content.setdefault(type, {}) + def getTrackersPerProtocol(self, type="shared", working_only=False): + if not self.site_announcer: + return None + + trackers = self.getTrackers(type) + + trackers_per_protocol = {} + for tracker_address in trackers: + protocol = self.getNormalizedTrackerProtocol(tracker_address) + if not protocol: + continue + if not working_only or self.isTrackerWorking(tracker_address, type=type): + trackers_per_protocol.setdefault(protocol, []).append(tracker_address) + + return trackers_per_protocol + def getWorkingTrackers(self, type="shared"): trackers = { key: tracker for key, tracker in self.getTrackers(type).items() - if tracker["time_success"] > time.time() - 60 * 60 + if self.isTrackerWorking(key, type) } return trackers @@ -133,8 +193,34 @@ def save(self): helper.atomicWrite(self.file_path, json.dumps(self.file_content, indent=2, sort_keys=True).encode("utf8")) self.log.debug("Saved in %.3fs" % (time.time() - s)) + def enoughWorkingTrackers(self, type="shared"): + supported_protocols = self.getSupportedProtocols() + if not supported_protocols: + return False + + trackers_per_protocol = self.getTrackersPerProtocol(type="shared", working_only=True) + if not trackers_per_protocol: + return False + + total_nr = 0 + + for protocol in supported_protocols: + trackers = trackers_per_protocol.get(protocol, []) + if len(trackers) < config.working_shared_trackers_limit_per_protocol: + self.log.debug("Not enough working trackers for protocol %s: %s < %s" % ( + protocol, len(trackers), config.working_shared_trackers_limit_per_protocol)) + return False + total_nr += len(trackers) + + if total_nr < config.working_shared_trackers_limit: + self.log.debug("Not enough working trackers: %s < %s" % ( + total_nr, config.working_shared_trackers_limit)) + return False + + return True + def discoverTrackers(self, peers): - if len(self.getWorkingTrackers()) > config.working_shared_trackers_limit: + if not self.enoughWorkingTrackers(type="shared"): return False s = time.time() num_success = 0 @@ -215,6 +301,7 @@ def portCheck(self, *args, **kwargs): class ConfigPlugin(object): def createArguments(self): group = self.parser.add_argument_group("AnnounceShare plugin") - group.add_argument('--working_shared_trackers_limit', help='Stop discovering new shared trackers after this number of shared trackers reached', default=5, type=int, metavar='limit') + group.add_argument('--working_shared_trackers_limit', help='Stop discovering new shared trackers after this number of shared trackers reached (total)', default=5, type=int, metavar='limit') + group.add_argument('--working_shared_trackers_limit_per_protocol', help='Stop discovering new shared trackers after this number of shared trackers reached per each supported protocol', default=3, type=int, metavar='limit') return super(ConfigPlugin, self).createArguments() From 9576650e26eb1aa82dd100a25477310e65171f9d Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 3 Jul 2019 13:51:39 +0700 Subject: [PATCH 04/14] AnnounceShare: replace the magic constants (60 * 60) with meaningful names --- plugins/AnnounceShare/AnnounceSharePlugin.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/AnnounceShare/AnnounceSharePlugin.py b/plugins/AnnounceShare/AnnounceSharePlugin.py index a1c9faabc..f8c551ffa 100644 --- a/plugins/AnnounceShare/AnnounceSharePlugin.py +++ b/plugins/AnnounceShare/AnnounceSharePlugin.py @@ -15,6 +15,10 @@ class TrackerStorage(object): def __init__(self): self.site_announcer = None self.log = logging.getLogger("TrackerStorage") + + self.working_tracker_time_interval = 60 * 60 + self.tracker_down_time_interval = 60 * 60 + self.file_path = "%s/trackers.json" % config.data_dir self.load() self.time_discover = 0.0 @@ -128,7 +132,7 @@ def onTrackerError(self, tracker_address): error_limit = 30 error_limit - if trackers[tracker_address]["num_error"] > error_limit and trackers[tracker_address]["time_success"] < time.time() - 60 * 60: + if trackers[tracker_address]["num_error"] > error_limit and trackers[tracker_address]["time_success"] < time.time() - self.tracker_down_time_interval: self.log.debug("Tracker %s looks down, removing." % tracker_address) del trackers[tracker_address] @@ -138,7 +142,7 @@ def isTrackerWorking(self, tracker_address, type="shared"): if not tracker: return False - if tracker["time_success"] > time.time() - 60 * 60: + if tracker["time_success"] > time.time() - self.working_tracker_time_interval: return True return False From 3d1328014bfd033f59e0b19087b589d23d2058f4 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 3 Jul 2019 14:02:04 +0700 Subject: [PATCH 05/14] AnnounceShare: Don't drop trackers from the list, if there haven't been any successful announces recently, to protect from the false positives during network connectivity issues. --- plugins/AnnounceShare/AnnounceSharePlugin.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugins/AnnounceShare/AnnounceSharePlugin.py b/plugins/AnnounceShare/AnnounceSharePlugin.py index f8c551ffa..7f452d70a 100644 --- a/plugins/AnnounceShare/AnnounceSharePlugin.py +++ b/plugins/AnnounceShare/AnnounceSharePlugin.py @@ -22,6 +22,7 @@ def __init__(self): self.file_path = "%s/trackers.json" % config.data_dir self.load() self.time_discover = 0.0 + self.time_success = 0.0 atexit.register(self.save) def setSiteAnnouncer(self, site_announcer): @@ -118,6 +119,8 @@ def onTrackerSuccess(self, tracker_address, latency): trackers[tracker_address]["time_success"] = time.time() trackers[tracker_address]["num_error"] = 0 + self.time_success = time.time() + def onTrackerError(self, tracker_address): trackers = self.getTrackers() if tracker_address not in trackers: @@ -126,6 +129,11 @@ def onTrackerError(self, tracker_address): trackers[tracker_address]["time_error"] = time.time() trackers[tracker_address]["num_error"] += 1 + if self.time_success < time.time() - self.tracker_down_time_interval / 2: + # Don't drop trackers from the list, if there haven't been any successful announces recently. + # There may be network connectivity issues. + return + if len(self.getWorkingTrackers()) >= config.working_shared_trackers_limit: error_limit = 5 else: From e716164be43827a781660fddb2b75a658cb8dd46 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 3 Jul 2019 16:16:16 +0700 Subject: [PATCH 06/14] AnnounceShare: fix a typo (= vs ==) --- plugins/AnnounceShare/AnnounceSharePlugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/AnnounceShare/AnnounceSharePlugin.py b/plugins/AnnounceShare/AnnounceSharePlugin.py index 7f452d70a..49e8128a8 100644 --- a/plugins/AnnounceShare/AnnounceSharePlugin.py +++ b/plugins/AnnounceShare/AnnounceSharePlugin.py @@ -63,7 +63,7 @@ def getNormalizedTrackerProtocol(self, tracker_address): protocol = address_parts["protocol"] if protocol == "https": - protocol == "http" + protocol = "http" return protocol From acc3f836bb5ebff01e7f305e7bc6a235af54fa45 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 3 Jul 2019 17:45:01 +0700 Subject: [PATCH 07/14] AnnounceShare: fix a typo in discoverTrackers() --- plugins/AnnounceShare/AnnounceSharePlugin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/AnnounceShare/AnnounceSharePlugin.py b/plugins/AnnounceShare/AnnounceSharePlugin.py index 49e8128a8..28a047eca 100644 --- a/plugins/AnnounceShare/AnnounceSharePlugin.py +++ b/plugins/AnnounceShare/AnnounceSharePlugin.py @@ -232,8 +232,11 @@ def enoughWorkingTrackers(self, type="shared"): return True def discoverTrackers(self, peers): - if not self.enoughWorkingTrackers(type="shared"): + if self.enoughWorkingTrackers(type="shared"): return False + + self.log.debug("Discovering trackers from %s peers..." % len(peers)) + s = time.time() num_success = 0 for peer in peers: From a61df634e43cb2eaca46c7a397c1e4e9727c5884 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 3 Jul 2019 18:14:38 +0700 Subject: [PATCH 08/14] AnnounceShare: rewrite getSupportedProtocols() using set(). Thanks to @imachug. --- plugins/AnnounceShare/AnnounceSharePlugin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/AnnounceShare/AnnounceSharePlugin.py b/plugins/AnnounceShare/AnnounceSharePlugin.py index 28a047eca..960d4b1b2 100644 --- a/plugins/AnnounceShare/AnnounceSharePlugin.py +++ b/plugins/AnnounceShare/AnnounceSharePlugin.py @@ -73,14 +73,14 @@ def getSupportedProtocols(self): supported_trackers = self.site_announcer.getSupportedTrackers() - protocols = {} + protocols = set() for tracker_address in supported_trackers: protocol = self.getNormalizedTrackerProtocol(tracker_address) if not protocol: continue - protocols[protocol] = True + protocols.add(protocol) - protocols = list(protocols.keys()) + protocols = list(protocols) self.log.debug("Supported tracker protocols: %s" % protocols) From 522abb1f2dfb11a1cdab4c4aaa72cd5dce6a0fa9 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 3 Jul 2019 18:19:02 +0700 Subject: [PATCH 09/14] AnnounceShare: replace one more magic constant with a meaningful name --- plugins/AnnounceShare/AnnounceSharePlugin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/AnnounceShare/AnnounceSharePlugin.py b/plugins/AnnounceShare/AnnounceSharePlugin.py index 960d4b1b2..564e6be5a 100644 --- a/plugins/AnnounceShare/AnnounceSharePlugin.py +++ b/plugins/AnnounceShare/AnnounceSharePlugin.py @@ -18,6 +18,7 @@ def __init__(self): self.working_tracker_time_interval = 60 * 60 self.tracker_down_time_interval = 60 * 60 + self.tracker_discover_time_interval = 5 * 60 self.file_path = "%s/trackers.json" % config.data_dir self.load() @@ -272,7 +273,7 @@ def discoverTrackers(self, peers): class SiteAnnouncerPlugin(object): def getTrackers(self): tracker_storage.setSiteAnnouncer(self) - if tracker_storage.time_discover < time.time() - 5 * 60: + if tracker_storage.time_discover < time.time() - tracker_storage.tracker_discover_time_interval: tracker_storage.time_discover = time.time() gevent.spawn(tracker_storage.discoverTrackers, self.site.getConnectedPeers()) trackers = super(SiteAnnouncerPlugin, self).getTrackers() From 6afd430907037d003b0cfdffc0d8c46bb7079095 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 3 Jul 2019 18:27:53 +0700 Subject: [PATCH 10/14] AnnounceShare: increase the default limits, so that the minimum number of zero:// trackers still were 5, as before the redesign --- plugins/AnnounceShare/AnnounceSharePlugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/AnnounceShare/AnnounceSharePlugin.py b/plugins/AnnounceShare/AnnounceSharePlugin.py index 564e6be5a..408266a7c 100644 --- a/plugins/AnnounceShare/AnnounceSharePlugin.py +++ b/plugins/AnnounceShare/AnnounceSharePlugin.py @@ -317,7 +317,7 @@ def portCheck(self, *args, **kwargs): class ConfigPlugin(object): def createArguments(self): group = self.parser.add_argument_group("AnnounceShare plugin") - group.add_argument('--working_shared_trackers_limit', help='Stop discovering new shared trackers after this number of shared trackers reached (total)', default=5, type=int, metavar='limit') - group.add_argument('--working_shared_trackers_limit_per_protocol', help='Stop discovering new shared trackers after this number of shared trackers reached per each supported protocol', default=3, type=int, metavar='limit') + group.add_argument('--working_shared_trackers_limit', help='Stop discovering new shared trackers after this number of shared trackers reached (total)', default=10, type=int, metavar='limit') + group.add_argument('--working_shared_trackers_limit_per_protocol', help='Stop discovering new shared trackers after this number of shared trackers reached per each supported protocol', default=5, type=int, metavar='limit') return super(ConfigPlugin, self).createArguments() From 209bf62be897fb6ea65493708e3fa4f6bdc55376 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Thu, 4 Jul 2019 00:11:46 +0700 Subject: [PATCH 11/14] AnnounceShare: Add cleanup of trackers that are potentially supported but unused (forbidden by the settings etc) --- plugins/AnnounceShare/AnnounceSharePlugin.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/plugins/AnnounceShare/AnnounceSharePlugin.py b/plugins/AnnounceShare/AnnounceSharePlugin.py index 408266a7c..c66bb9a0d 100644 --- a/plugins/AnnounceShare/AnnounceSharePlugin.py +++ b/plugins/AnnounceShare/AnnounceSharePlugin.py @@ -74,6 +74,17 @@ def getSupportedProtocols(self): supported_trackers = self.site_announcer.getSupportedTrackers() + # If a tracker is in our list, but is absent from the results of getSupportedTrackers(), + # it seems to be supported by software, but forbidden by the settings or network configuration. + # We check and remove thoose trackers here, since onTrackerError() is never emitted for them. + trackers = self.getTrackers() + for tracker_address, tracker in list(trackers.items()): + t = max(trackers[tracker_address]["time_added"], + trackers[tracker_address]["time_success"]) + if tracker_address not in supported_trackers and t < time.time() - self.tracker_down_time_interval: + self.log.debug("Tracker %s looks unused, removing." % tracker_address) + del trackers[tracker_address] + protocols = set() for tracker_address in supported_trackers: protocol = self.getNormalizedTrackerProtocol(tracker_address) From d3a51dd0c7786e2c7fc238af81e12c3ed5f0fa9d Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Thu, 4 Jul 2019 00:24:50 +0700 Subject: [PATCH 12/14] AnnounceShare: Shuffle both sent and received tracker lists for the more uniform filling of per-protocol buckets. --- plugins/AnnounceShare/AnnounceSharePlugin.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/AnnounceShare/AnnounceSharePlugin.py b/plugins/AnnounceShare/AnnounceSharePlugin.py index c66bb9a0d..131d9f8ec 100644 --- a/plugins/AnnounceShare/AnnounceSharePlugin.py +++ b/plugins/AnnounceShare/AnnounceSharePlugin.py @@ -1,3 +1,4 @@ +import random import time import os import logging @@ -260,6 +261,8 @@ def discoverTrackers(self, peers): continue num_success += 1 + + random.shuffle(res["trackers"]) for tracker_address in res["trackers"]: if type(tracker_address) is bytes: # Backward compatibilitys tracker_address = tracker_address.decode("utf8") @@ -310,6 +313,7 @@ def announceTracker(self, tracker, *args, **kwargs): class FileRequestPlugin(object): def actionGetTrackers(self, params): shared_trackers = list(tracker_storage.getWorkingTrackers("shared").keys()) + random.shuffle(shared_trackers) self.response({"trackers": shared_trackers}) From 61864ed2ab3205b3574c4b86215cfe095ff6e7de Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Thu, 4 Jul 2019 01:00:19 +0700 Subject: [PATCH 13/14] AnnounceShare: take account of the per-protocol limit in onTrackerError() --- plugins/AnnounceShare/AnnounceSharePlugin.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/plugins/AnnounceShare/AnnounceSharePlugin.py b/plugins/AnnounceShare/AnnounceSharePlugin.py index 131d9f8ec..aed413246 100644 --- a/plugins/AnnounceShare/AnnounceSharePlugin.py +++ b/plugins/AnnounceShare/AnnounceSharePlugin.py @@ -147,11 +147,16 @@ def onTrackerError(self, tracker_address): # There may be network connectivity issues. return - if len(self.getWorkingTrackers()) >= config.working_shared_trackers_limit: - error_limit = 5 - else: - error_limit = 30 - error_limit + protocol = self.getNormalizedTrackerProtocol(tracker_address) or "" + + nr_working_trackers_for_protocol = len(self.getTrackersPerProtocol(working_only=True).get(protocol, [])) + nr_working_trackers = len(self.getWorkingTrackers()) + + error_limit = 30 + if nr_working_trackers_for_protocol >= config.working_shared_trackers_limit_per_protocol: + error_limit = 10 + if nr_working_trackers >= config.working_shared_trackers_limit: + error_limit = 5 if trackers[tracker_address]["num_error"] > error_limit and trackers[tracker_address]["time_success"] < time.time() - self.tracker_down_time_interval: self.log.debug("Tracker %s looks down, removing." % tracker_address) From f64e57d2c26fd8086b1d70e36fe7b6e4f49b591f Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Thu, 4 Jul 2019 23:59:59 +0700 Subject: [PATCH 14/14] AnnounceShare: allow setting separate limits for each protocol; change the defaults: zero => 5 trackers, any other protocol => 2 trackers --- plugins/AnnounceShare/AnnounceSharePlugin.py | 30 ++++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/plugins/AnnounceShare/AnnounceSharePlugin.py b/plugins/AnnounceShare/AnnounceSharePlugin.py index aed413246..ef877ee43 100644 --- a/plugins/AnnounceShare/AnnounceSharePlugin.py +++ b/plugins/AnnounceShare/AnnounceSharePlugin.py @@ -4,6 +4,7 @@ import logging import json import atexit +import re import gevent @@ -21,16 +22,35 @@ def __init__(self): self.tracker_down_time_interval = 60 * 60 self.tracker_discover_time_interval = 5 * 60 + self.working_shared_trackers_limit_per_protocol = {} + self.working_shared_trackers_limit_per_protocol["other"] = 2 + self.file_path = "%s/trackers.json" % config.data_dir self.load() self.time_discover = 0.0 self.time_success = 0.0 atexit.register(self.save) + def initTrackerLimitForProtocol(self): + for s in re.split("[,;]", config.working_shared_trackers_limit_per_protocol): + x = s.split("=", 1) + if len(x) == 1: + x = ["other", x[0]] + try: + self.working_shared_trackers_limit_per_protocol[x[0]] = int(x[1]) + except ValueError: + pass + self.log.debug("Limits per protocol: %s" % self.working_shared_trackers_limit_per_protocol) + + def getTrackerLimitForProtocol(self, protocol): + l = self.working_shared_trackers_limit_per_protocol + return l.get(protocol, l.get("other")) + def setSiteAnnouncer(self, site_announcer): - if self.site_announcer == site_announcer: + if self.site_announcer: return self.site_announcer = site_announcer + self.initTrackerLimitForProtocol() self.recheckValidTrackers() def isTrackerAddressValid(self, tracker_address): @@ -153,7 +173,7 @@ def onTrackerError(self, tracker_address): nr_working_trackers = len(self.getWorkingTrackers()) error_limit = 30 - if nr_working_trackers_for_protocol >= config.working_shared_trackers_limit_per_protocol: + if nr_working_trackers_for_protocol >= self.getTrackerLimitForProtocol(protocol): error_limit = 10 if nr_working_trackers >= config.working_shared_trackers_limit: error_limit = 5 @@ -236,9 +256,9 @@ def enoughWorkingTrackers(self, type="shared"): for protocol in supported_protocols: trackers = trackers_per_protocol.get(protocol, []) - if len(trackers) < config.working_shared_trackers_limit_per_protocol: + if len(trackers) < self.getTrackerLimitForProtocol(protocol): self.log.debug("Not enough working trackers for protocol %s: %s < %s" % ( - protocol, len(trackers), config.working_shared_trackers_limit_per_protocol)) + protocol, len(trackers), self.getTrackerLimitForProtocol(protocol))) return False total_nr += len(trackers) @@ -338,6 +358,6 @@ class ConfigPlugin(object): def createArguments(self): group = self.parser.add_argument_group("AnnounceShare plugin") group.add_argument('--working_shared_trackers_limit', help='Stop discovering new shared trackers after this number of shared trackers reached (total)', default=10, type=int, metavar='limit') - group.add_argument('--working_shared_trackers_limit_per_protocol', help='Stop discovering new shared trackers after this number of shared trackers reached per each supported protocol', default=5, type=int, metavar='limit') + group.add_argument('--working_shared_trackers_limit_per_protocol', help='Stop discovering new shared trackers after this number of shared trackers reached per each supported protocol', default="zero=5,other=2", metavar='limit') return super(ConfigPlugin, self).createArguments()