Add: detect loop in analyzer, bump dependencies (#402)

* Add: detect loop in analyzer

* Change: use skip page error

* Bump deps

Author: eight04

comiccrawler/analyzer.py

@@ -102,8 +102,11 @@ def analyze_pages(self):
 		url = self.mission.url
 		old_eps = EpisodeList(self.mission.episodes or ())
 		new_eps = EpisodeList()
-		
+
+		visited_urls = set()
+
 		while True:
+			visited_urls.add(url)
 			try:
 				eps = list(self.mission.module.get_episodes(self.html, url))
 			except SkipPageError:
@@ -134,6 +137,8 @@ def analyze_pages(self):
 			next_url = self.get_next_page(self.html, url)
 			if not next_url:
 				break
+			if next_url in visited_urls:
+				raise TypeError("Loop detected: {}".format(next_url))
 			url = next_url
 			print('Analyzing {}...'.format(url))
 			sleep(getattr(self.mission.module, "rest_analyze", 0))

comiccrawler/mods/exh.py

@@ -36,6 +36,7 @@ def has_login(html, page="ep"):
 	return True
 
 def check_login(html, page="ep"):
+	# breakpoint()
 	if not has_login(html, page):
 		raise PauseDownloadError("You didn't login!")
 

comiccrawler/mods/twitter.py

@@ -11,7 +11,7 @@
 from ..episode import Episode
 from ..grabber import grabber
 from ..url import update_qs
-from ..error import is_http, SkipEpisodeError
+from ..error import is_http, SkipEpisodeError, SkipPageError
 from ..session_manager import session_manager
 from ..util import get_cookie
 
@@ -73,7 +73,7 @@ def get_episodes(html, url):
 
 		endpoint = user_media_graph if is_media(url) else user_tweets_graph
 		next_page_cache[url] = endpoint(userId=uid)
-		return
+		raise SkipPageError
 
 	if any(k in url for k in ["UserTweets", "UserMedia"]):
 		data = json.loads(html)
@@ -84,7 +84,7 @@ def get_episodes(html, url):
 				extract_pin_entry(instruction["entry"], url)
 
 			if instruction["type"] == "TimelineAddEntries":
-				yield from reversed(list(extract_added_entries(instruction["entries"], url)))
+				return reversed(list(extract_added_entries(instruction["entries"], url)))
 
 def tweet_result_to_episode(tweet_result):
 	try:

requirements-lock.txt

@@ -1,43 +1,45 @@
 ansicon==1.89.0
-astroid==3.3.5
+astroid==3.3.9
 belfrywidgets==1.0.3
 bidict==0.23.1
 blessed==1.20.0
 Brotli==1.1.0
-certifi==2024.8.30
+certifi==2025.1.31
 cffi==1.17.1
-charset-normalizer==3.4.0
+charset-normalizer==3.4.1
 colorama==0.4.6
-curl_cffi==0.7.3
+curl_cffi==0.10.0
 deno_vm==0.6.0
 desktop3==0.5.3
 dill==0.3.9
 docopt==0.6.2
 docutils==0.21.2
-enlighten==1.12.4
+enlighten==1.14.1
+id==1.5.0
 idna==3.10
 importlib_metadata==8.5.0
-isort==5.13.2
+isort==6.0.1
 jaraco.classes==3.4.0
 jaraco.context==6.0.1
 jaraco.functools==4.1.0
 jinxed==1.3.0
-keyring==25.5.0
-livereload==2.7.0
+keyring==25.6.0
+livereload==2.7.1
 markdown-it-py==3.0.0
 mccabe==0.7.0
 mdurl==0.1.2
-more-itertools==10.5.0
-nh3==0.2.18
+more-itertools==10.6.0
+nh3==0.2.21
 ordered-set==3.1.1
+packaging==24.2
 pkginfo==1.10.0
 platformdirs==4.3.6
 prefixed==0.9.0
 puremagic==1.28
 pycparser==2.22
 pycryptodomex==3.21.0
-Pygments==2.18.0
-pylint==3.3.1
+Pygments==2.19.1
+pylint==3.3.5
 pyperclip==1.9.0
 pythreadworker==0.10.0
 pywin32-ctypes==0.2.3
@@ -50,15 +52,15 @@ rich==13.9.4
 safeprint==0.2.0
 semver==2.13.0
 Send2Trash==1.8.3
-setuptools==75.5.0
-six==1.16.0
+setuptools==76.0.0
+six==1.17.0
 tomlkit==0.13.2
-tornado==6.4.1
-twine==5.1.1
+tornado==6.4.2
+twine==6.1.0
 typing_extensions==4.8.0
 uncurl==0.0.11
-urllib3==2.2.3
+urllib3==2.3.0
 wcwidth==0.2.13
 win_unicode_console==0.5
-yt-dlp==2024.11.4
+yt-dlp==2025.2.19
 zipp==3.21.0

requirements.txt

@@ -1,6 +1,6 @@
-certifi==2024.8.30
+certifi==2025.1.31
 docutils==0.21.2
-pygments==2.18.0
-pylint==3.3.1
+pygments==2.19.1
+pylint==3.3.5
 pyxcute==0.8.1
-twine==5.1.1
+twine==6.1.0

setup.cfg

@@ -31,18 +31,18 @@ install_requires =
     belfrywidgets~=1.0
     bidict~=0.23.1
     brotli~=1.1
-    curl_cffi~=0.7.3
+    curl_cffi~=0.10.0
     deno-vm~=0.6.0
     desktop3~=0.5.3
     docopt~=0.6.2
-    enlighten~=1.12
+    enlighten~=1.14
     puremagic~=1.28
     pycryptodomex~=3.21
     pythreadworker~=0.10.0
     safeprint~=0.2.0
     uncurl~=0.0.11
-    urllib3~=2.2
-    yt-dlp~=2024.11
+    urllib3~=2.3
+    yt-dlp~=2025.2
 
 python_requires = >=3.10