Update http.py & requirements.txt (#27)

* Update http.py

Remove external parameter (not used anymore).

* Update requirements.txt

Upgrade to v1.2.1

---------

Co-authored-by: Changxin Miao <[email protected]>

Author: melroy89

page_content_extractor/http.py

@@ -51,8 +51,8 @@ def init_poolmanager(self, *args, **kwargs):
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 # Some sites just refuses bot connection
-ua = UserAgent(use_external_data=True, browsers=['chrome'], fallback='Twitterbot/1.0')
-ua_str = ua.data_browsers['chrome'][0]  # Use the latest, in case of "unsupported browser" error
+ua = UserAgent(browsers=['chrome'], fallback='Twitterbot/1.0')
+ua_str = ua.random
 logger.info(f'Use user-agent {ua_str}')
 
 session = requests.Session()

requirements.txt

@@ -11,7 +11,7 @@ mock==5.1.0
 werkzeug==2.3.7
 feedwerk==1.1.0
 Pillow==10.0.0
-fake-useragent==1.1.3  # 1.2.0 is a breaking change, and doesnot work properly
+fake-useragent==1.2.1
 olefile
 openai==0.28.0
 torch==2.0.1
@@ -22,4 +22,4 @@ python_slugify==8.0.1
 sqlalchemy==2.0.20
 psycopg2==2.9.7
 humanize==4.8.0
-llama-cpp-python==0.1.84
+llama-cpp-python==0.1.84