Fix sitemap issue and add a sitmap limit, fix playground (#43)

Author: vegito22

apps/tasks.py

@@ -134,7 +134,11 @@ def extract_urls_task(url):
     url_content_type = get_url_content_type(url=url)
     url_content_type_parts = url_content_type.split(';')
     mime_type = url_content_type_parts[0]
-    if mime_type != 'text/html' or is_youtube_video_url(url):
+    
+    if is_youtube_video_url(url):
+        return [url]
+    
+    if mime_type != 'text/html' and not is_sitemap_url(url):
         return [url]
 
     # Get url domain

common/utils/crawlers.py

@@ -11,6 +11,7 @@
 from scrapy.spiders import Rule
 from scrapy.spiders import SitemapSpider
 from unstructured.partition.auto import partition_html
+from scrapy.exceptions import CloseSpider
 
 from django.conf import settings
 
@@ -32,13 +33,17 @@ def get_domain(url):
 class SitemapXMLSpider(SitemapSpider):
     name = 'sitemap_spider'
 
-    def __init__(self, url, output, *args, **kwargs):
+    def __init__(self, url, output, max_urls = 20, *args, **kwargs):
         self.sitemap_urls = [url]
         self.output = output
+        self.max_urls = max_urls
         super(SitemapXMLSpider, self).__init__(*args, **kwargs)
 
     def parse(self, response):
         data = {}
+        if len(self.output) > self.max_urls:
+            raise CloseSpider('Reached maximum number of crawled URLs')
+            
         # Extract data from the page using CSS or XPath selectors
         data['title'] = response.css('title::text').get()
         data['url'] = response.url

common/utils/utils.py

@@ -143,7 +143,7 @@ def get_url_content_type(url):
 def is_sitemap_url(url):
     try:
         content_type = get_url_content_type(url)
-        if 'application/xml' in content_type or 'text/xml' in content_type or 'text/plain' in content_type:
+        if 'application/xml' in content_type or 'text/xml' in content_type or 'text/plain' in content_type or 'application/rss+xml' in content_type:
             return True
         else:
             return False

processors/apis.py

@@ -454,7 +454,10 @@ async def stream_output():
             logger.exception(e)
             raise Exception(f'Error starting coordinator: {e}')
 
-        return {'output': json.loads(output)} if 'errors' not in output else output
+        if isinstance(output, dict) and 'errors' in output:
+            return output
+        else:
+            return {'output': json.loads(output)}
 
 
 class ApiProviderViewSet(viewsets.ViewSet):