Catch exceptions in all callbacks. Treat www subdomain the same as no

subdomain. Fixed possible invalid URL schemes when the <base> element
was used.

Author: tijme

docs/source/options_crawling_scope.rst

@@ -45,6 +45,8 @@ Only crawl pages with the same protocol as the startpoint (e.g. only https) if T
 
 Only crawl pages with the same subdomain as the startpoint if True. If the startpoint is not a subdomain, no subdomains will be crawled. Default is True.
 
+Please note that the `www` subdomain will be treated the same as no subdomain.
+
 .. code:: python
 
     options.scope.subdomain_must_match = True

nyawc/Crawler.py

@@ -139,7 +139,10 @@ def __crawler_start(self):
 
         """
 
-        self.__options.callbacks.crawler_before_start()
+        try:
+            self.__options.callbacks.crawler_before_start()
+        except Exception as e:
+            print(e)
 
         self.__spawn_new_requests()
 
@@ -175,7 +178,10 @@ def __crawler_stop(self):
     def __crawler_finish(self):
         """Called when the crawler is finished because there are no queued requests left or it was stopped."""
 
-        self.__options.callbacks.crawler_after_finish(self.queue)
+        try:
+            self.__options.callbacks.crawler_after_finish(self.queue)
+        except Exception as e:
+            print(e)
 
     def __request_start(self, queue_item):
         """Execute the request in given queue item.
@@ -185,7 +191,11 @@ def __request_start(self, queue_item):
 
         """
 
-        action = self.__options.callbacks.request_before_start(self.queue, queue_item)
+        try:
+            action = self.__options.callbacks.request_before_start(self.queue, queue_item)
+        except Exception as e:
+            action = None
+            print(e)
 
         if action == CrawlerActions.DO_STOP_CRAWLING:
             self.__should_stop = True
@@ -223,7 +233,11 @@ def __request_finish(self, queue_item, new_requests, request_failed=False):
             new_queue_items = self.__add_scraped_requests_to_queue(queue_item, new_requests)
             self.queue.move(queue_item, QueueItem.STATUS_FINISHED)
 
-        action = self.__options.callbacks.request_after_finish(self.queue, queue_item, new_queue_items)
+        try:
+            action = self.__options.callbacks.request_after_finish(self.queue, queue_item, new_queue_items)
+        except Exception as e:
+            action = None
+            print(e)
 
         if action == CrawlerActions.DO_STOP_CRAWLING:
             self.__should_stop = True

nyawc/helpers/HTTPRequestHelper.py

@@ -82,8 +82,13 @@ def complies_with_scope(queue_item, new_request, scope):
                 return False
 
         if scope.subdomain_must_match:
-            if URLHelper.get_subdomain(queue_item.request.url) != URLHelper.get_subdomain(new_request.url):
-                return False
+            current_subdomain = URLHelper.get_subdomain(queue_item.request.url)
+            new_subdomain = URLHelper.get_subdomain(new_request.url)
+
+            if current_subdomain != new_subdomain:
+                if current_subdomain != "www" and new_subdomain != "":
+                    if new_subdomain != "www" and current_subdomain != "":
+                        return False
 
         if scope.hostname_must_match:
             if URLHelper.get_hostname(queue_item.request.url) != URLHelper.get_hostname(new_request.url):

nyawc/scrapers/HTMLSoupLinkScraper.py

@@ -63,7 +63,7 @@ def derived_get_requests(self):
         # Always use the URL from the base element if it exists.
         # https://www.w3schools.com/tags/tag_base.asp
         if base_element:
-            host = base_element["href"]
+            host = URLHelper.make_absolute(host, base_element["href"])
 
         found_requests = []