feat: refactoring of mdscraper

Author: VinciGit00

examples/openai/md_scraper_openai.py

@@ -37,7 +37,7 @@
 # ************************************************
 
 md_scraper_graph = DocumentScraperGraph(
-    prompt="List me all the authors, title and genres of the books",
+    prompt="List me all the projects",
     source=text,  # Pass the content of the file, not the file object
     config=graph_config
 )

scrapegraphai/nodes/parse_node.py

@@ -85,10 +85,13 @@ def execute(self, state: dict) -> dict:
         else:
             docs_transformed = docs_transformed[0]
 
-            link_urls, img_urls = self._extract_urls(docs_transformed.page_content, source)
+            try:
+                link_urls, img_urls = self._extract_urls(docs_transformed.page_content, source)
+            except Exception as e:
+                link_urls, img_urls = "", ""
 
             chunk_size = self.chunk_size
-            chunk_size = min(chunk_size - 500, int(chunk_size * 0.75))
+            chunk_size = min(chunk_size - 500, int(chunk_size * 0.8))
 
             if isinstance(docs_transformed, Document):
                 chunks = split_text_into_chunks(text=docs_transformed.page_content,