Getting error while using recursive url loader

[deepak-hl]

Checked other resources

• I added a very descriptive title to this question.
• I searched the LangChain documentation with the integrated search.
• I used the GitHub search to find a similar question and didn't find it.

Commit to Help

• I commit to help with one of those options 👆

Example Code

loader = MultiUrlRecursiveLoader(
            urls=all_urls,
            max_depth=250,
            extractor=lambda x: detect_and_parse(x)  # Use the appropriate parser dynamically
        )
        # texts = load_and_split_documents(loader, chunk_size=2000, chunk_overlap=50)
        urlDocument = loader.load()
        text_splitter = TokenTextSplitter(chunk_size=2000, chunk_overlap=50)
        texts = text_splitter.split_documents(documents=urlDocument)

class MultiUrlRecursiveLoader(RecursiveUrlLoader):
    def __init__(
        self,
        urls: List[str],
        max_depth: Optional[int] = 2,
        use_async: Optional[bool] = None,
        extractor: Optional[Callable[[str], str]] = None,
        metadata_extractor: Optional[Callable[[str, str, requests.Response], dict]] = None,
        exclude_dirs: Optional[Sequence[str]] = (),
        timeout: Optional[int] = 100,
        prevent_outside: bool = True,
        link_regex: Union[str, re.Pattern, None] = None,
        headers: Optional[dict] = None,
        check_response_status: bool = False,
        continue_on_failure: bool = True,
        # base_url: Optional[str] = None,
        # autoset_encoding: bool = True,
        # encoding: Optional[str] = None,
    ) -> None:
        self.urls = urls
        super().__init__(
            url=urls[0],  # Pass the first URL to the parent class
            max_depth=max_depth,
            use_async=use_async,
            extractor=extractor,
            metadata_extractor=metadata_extractor,
            exclude_dirs=exclude_dirs,
            timeout=timeout,
            prevent_outside=prevent_outside,
            link_regex=link_regex,
            headers=headers,
            check_response_status=check_response_status,
            continue_on_failure=continue_on_failure,
            # base_url=base_url,
            # autoset_encoding=autoset_encoding,
            # encoding=encoding,
        )

    def lazy_load(self) -> Iterator[Document]:
        visited: Set[str] = set()
        results = []
        for url in self.urls:
            if self.use_async:
                results.extend(asyncio.run(self._async_get_child_links_recursive(url, visited)))
            else:
                results.extend(self._get_child_links_recursive(url, visited))
        return iter(results or [])


def detect_and_parse(content):
    # If content starts with an XML declaration, use XML parser
    if content.strip().startswith('<?xml'):
        return Soup(content, 'xml')  # Return Soup object to allow tag-based operations
    else:
        # Otherwise, use the HTML parser
        return Soup(content, 'html.parser')

Description

ValidationError(model='Document', errors=[{'loc': ('page_content',), 'msg': 'str type expected', 'type': 'type_error.str'}])

System Info

langchain

[deepak-hl]

@dosu

[eyurtsev]

Use this: https://github.com/langchain-ai/langchain/discussions/categories/ask-dosu

We're trying to separate the Q&A from dosu, so that it's easier to get help from people.

We also prefer the Q&A to be used for usage questions, not programming questions.

ValidationError(model='Document', errors=[{'loc': ('page_content',), 'msg': 'str type expected', 'type': 'type_error.str'}])

Tells you that a Document is being created without page_content.