File tree Expand file tree Collapse file tree 1 file changed +4
-3
lines changed Expand file tree Collapse file tree 1 file changed +4
-3
lines changed Original file line number Diff line number Diff line change 33"""
44from typing import List , Optional
55import re
6- from tqdm import tqdm
76from urllib .parse import urlparse , parse_qs
7+ from tqdm import tqdm
88from langchain .prompts import PromptTemplate
99from langchain_core .output_parsers import JsonOutputParser
1010from langchain_core .runnables import RunnableParallel
@@ -74,10 +74,11 @@ def _is_language_url(self, url):
7474 parsed_url = urlparse (url )
7575 query_params = parse_qs (parsed_url .query )
7676
77- return any (indicator in parsed_url .path .lower () or indicator in query_params for indicator in lang_indicators )
77+ return any (indicator in parsed_url .path .lower () \
78+ or indicator in query_params for indicator in lang_indicators )
7879 def _is_potentially_irrelevant (self , url ):
7980 if not self .filter_links :
80- return False
81+ return False
8182
8283 irrelevant_keywords = self .filter_config .get ("irrelevant_keywords" , [])
8384 return any (keyword in url .lower () for keyword in irrelevant_keywords )
You can’t perform that action at this time.
0 commit comments