Add: max_threads options

Author: eight04

comiccrawler/config.py

@@ -28,7 +28,8 @@ class Config:
 		"mission_conflict_action": "update",
 		"browser": "",
 		"browser_profile": "",
-		"max_errors": "10"
+		"max_errors": "10",
+		"max_threads": "3"
 	}
 	def __init__(self, path):
 		self.path = expanduser(path)

comiccrawler/download_manager.py

@@ -61,6 +61,7 @@ def __init__(self):
 		"""Construct."""
 		self.lock = Lock()
 		self.crawlers: bidict[ModuleType, Worker] = bidict()
+		self.max_threads: int = setting.getint("max_threads", 3)
 		self.mod_errors: dict[ModuleType, int] = defaultdict(int)
 		self.analyze_threads = ThreadSafeSet()
 		self.library_thread = None
@@ -130,7 +131,7 @@ def _(event):
 				self.mod_errors[mod] = 0
 				return
 
-			self.start_download_mod(mod)
+			self.start_download_all(continue_=True)
 
 		@thread.listen("DOWNLOAD_INVALID")
 		def _(event):
@@ -144,10 +145,11 @@ def start_download(self):
 		"""Start downloading."""
 		self.start_download_all()
 
-	def start_download_all(self):
+	def start_download_all(self, continue_=False):
 		missions = mission_manager.get_all("view", lambda m: m.state in ("ANALYZED", "PAUSE", "ERROR", "UPDATE"))
 		if not missions:
-			print("所有任務已下載完成")
+			if not continue_:
+				print("所有任務已下載完成")
 			return
 
 		for mission in missions:
@@ -163,6 +165,9 @@ def start_download_mod(self, mod):
 			if mod in self.crawlers:
 				return
 
+			if len(self.crawlers) >= self.max_threads:
+				return
+
 			def do_download():
 				debug_log("do_download")
 				with load_episodes(mission):
@@ -181,7 +186,7 @@ def stop_download(self):
 
 			# FIXME: shouldn't we wait for thread stop?
 			self.crawlers.clear()
-			print("Stop downloading")
+			print("Stop downloading, download_manager.crawlers cleared")
 
 	def start_analyze(self, mission, on_finished=None):
 		"""Start analyzing"""

comiccrawler/grabber.py

@@ -207,9 +207,9 @@ def _():
 						content_list.append(chunk)
 						counter.update(len(chunk))
 						loaded += len(chunk)
-	except WorkerExit:
+	finally:
+		# FIXME: is it safe to always close the connection?
 		r.close()
-		raise
 	if total and loaded < total:
 		raise IncompleteRead(loaded, total - loaded)
 	b = None

comiccrawler/mods/twitter.py

@@ -235,8 +235,8 @@ def variables(s):
 	)
 
 def errorhandler(err, crawler):
-	if is_http(err, 404) and crawler.image.url == err.request.url:
-		if match := re.match("(.*):orig", err.request.url):
+	if is_http(err, 404) and crawler.image.url == err.response.request.url:
+		if match := re.match("(.*):orig", err.response.request.url):
 			crawler.image.url = match.group(1)
 			return