refactor: centralize get_requests_session() in shared.get_session()

Author: Joyakis

scripts/1-fetch/arxiv_fetch.py

@@ -22,8 +22,6 @@
 from pygments import highlight
 from pygments.formatters import TerminalFormatter
 from pygments.lexers import PythonTracebackLexer
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
 
 # Add parent directory so shared can be imported
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -335,17 +333,7 @@ def initialize_all_data_files(args):
     initialize_data_file(FILE_ARXIV_AUTHOR_BUCKET, HEADER_AUTHOR_BUCKET)
 
 
-def get_requests_session():
-    """Create request session with retry logic"""
-    retry_strategy = Retry(
-        total=5,
-        backoff_factor=10,
-        status_forcelist=shared.STATUS_FORCELIST,
-    )
-    session = requests.Session()
-    session.headers.update({"User-Agent": shared.USER_AGENT})
-    session.mount("https://", HTTPAdapter(max_retries=retry_strategy))
-    return session
+session = shared.get_requests_session()
 
 
 def normalize_license_text(raw_text):
@@ -533,7 +521,7 @@ def query_arxiv(args):
     """
 
     LOGGER.info("Beginning to fetch results from ArXiv API")
-    session = get_requests_session()
+    session = shared.get_requests_session()
 
     results_per_iteration = 50
 

scripts/1-fetch/europeana_fetch.py

@@ -23,7 +23,6 @@
 from pygments import highlight
 from pygments.formatters import TerminalFormatter
 from pygments.lexers import PythonTracebackLexer
-from requests.adapters import HTTPAdapter, Retry
 
 # Add parent directory for shared imports
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -103,17 +102,7 @@ def parse_arguments():
     return args
 
 
-def get_requests_session():
-    """Create a requests session with retry."""
-    max_retries = Retry(
-        total=5, backoff_factor=10, status_forcelist=shared.STATUS_FORCELIST
-    )
-    session = requests.Session()
-    session.mount("https://", HTTPAdapter(max_retries=max_retries))
-    session.headers.update(
-        {"accept": "application/json", "User-Agent": shared.USER_AGENT}
-    )
-    return session
+session = shared.get_requests_session(accept_header="application/json")
 
 
 def simplify_legal_tool(legal_tool):
@@ -433,7 +422,7 @@ def main():
             "EUROPEANA_API_KEY not found in environment variables", 1
         )
 
-    session = get_requests_session()
+    session = shared.get_requests_session()
 
     # Fetch facet lists once, including counts
     providers_full = get_facet_list(session, "DATA_PROVIDER")

scripts/1-fetch/github_fetch.py

@@ -17,8 +17,6 @@
 from pygments import highlight
 from pygments.formatters import TerminalFormatter
 from pygments.lexers import PythonTracebackLexer
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
 
 # Add parent directory so shared can be imported
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -80,23 +78,11 @@ def check_for_completion():
         pass  # File may not be found without --enable-save, etc.
 
 
-def get_requests_session():
-    max_retries = Retry(
-        total=5,
-        backoff_factor=10,
-        status_forcelist=shared.STATUS_FORCELIST,
-    )
-    session = requests.Session()
-    session.mount("https://", HTTPAdapter(max_retries=max_retries))
-    headers = {
-        "accept": "application/vnd.github+json",
-        "User-Agent": shared.USER_AGENT,
-    }
-    if GH_TOKEN:
-        headers["authorization"] = f"Bearer {GH_TOKEN}"
-    session.headers.update(headers)
-
-    return session
+session = shared.get_requests_session(
+    headers={"accept": "application/vnd.github+json"},
+    auth_token=GH_TOKEN,
+    auth_prefix="Bearer",
+)
 
 
 def write_data(args, tool_data):
@@ -162,7 +148,7 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     check_for_completion()
-    session = get_requests_session()
+    session = shared.get_requests_session()
     tool_data = query_github(args, session)
     args = write_data(args, tool_data)
     args = shared.git_add_and_commit(

scripts/1-fetch/openverse_fetch.py

@@ -25,8 +25,6 @@
 from pygments import highlight
 from pygments.formatters import TerminalFormatter
 from pygments.lexers import PythonTracebackLexer
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
 
 # Add parent directory so shared can be imported
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -83,18 +81,7 @@ def parse_arguments():
     return args
 
 
-def get_requests_session():
-    max_retries = Retry(
-        total=5,
-        backoff_factor=10,
-        status_forcelist=shared.STATUS_FORCELIST,
-    )
-    session = requests.Session()
-    session.mount("https://", HTTPAdapter(max_retries=max_retries))
-    session.headers.update(
-        {"accept": "application/json", "User-Agent": shared.USER_AGENT}
-    )
-    return session
+session = shared.get_requests_session(accept_header="application/json")
 
 
 def get_all_sources_and_licenses(session, media_type):
@@ -225,7 +212,7 @@ def write_data(args, data):
 
 def main():
     args = parse_arguments()
-    session = get_requests_session()
+    session = shared.get_requests_session()
     LOGGER.info("Starting Openverse Fetch Script...")
     records = query_openverse(session)
     write_data(args, records)

scripts/1-fetch/wikipedia_fetch.py

@@ -13,12 +13,9 @@
 from operator import itemgetter
 
 # Third-party
-import requests
 from pygments import highlight
 from pygments.formatters import TerminalFormatter
 from pygments.lexers import PythonTracebackLexer
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
 
 # Add parent directory so shared can be imported
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -66,16 +63,7 @@ def parse_arguments():
     return args
 
 
-def get_requests_session():
-    max_retries = Retry(
-        total=5,
-        backoff_factor=10,
-        status_forcelist=shared.STATUS_FORCELIST,
-    )
-    session = requests.Session()
-    session.mount("https://", HTTPAdapter(max_retries=max_retries))
-    session.headers.update({"User-Agent": shared.USER_AGENT})
-    return session
+session = shared.get_requests_session()
 
 
 def write_data(args, tool_data):
@@ -173,7 +161,7 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    tool_data = query_wikipedia_languages(get_requests_session())
+    tool_data = query_wikipedia_languages(shared.get_requests_session())
     args = write_data(args, tool_data)
     args = shared.git_add_and_commit(
         args,

scripts/shared.py

@@ -5,8 +5,10 @@
 from datetime import datetime, timezone
 
 # Third-party
+import requests
 from git import InvalidGitRepositoryError, NoSuchPathError, Repo
 from pandas import PeriodIndex
+from requests.adapters import HTTPAdapter, Retry
 
 # Constants
 STATUS_FORCELIST = [
@@ -31,6 +33,34 @@ def __init__(self, message, exit_code=None):
         super().__init__(self.message)
 
 
+def get_requests_session(
+    accept_header: str | None = None,
+    auth_token: str | None = None,
+    mount_https: bool = True,
+) -> requests.Session:
+    """Create a reusable requests session with retry logic."""
+    retry_strategy = Retry(
+        total=5,
+        backoff_factor=10,
+        status_forcelist=STATUS_FORCELIST,
+    )
+
+    session = requests.Session()
+    session.mount("https://", HTTPAdapter(max_retries=retry_strategy))
+
+    headers = {"User-Agent": USER_AGENT}
+    if accept_header:
+        headers["accept"] = accept_header
+    if auth_token:
+        headers["authorization"] = f"Bearer {auth_token}"
+    # Mount retry adapter for HTTPS
+    if mount_https:
+        session.mount("https://", HTTPAdapter(max_retries=retry_strategy))
+
+    session.headers.update(headers)
+    return session
+
+
 def git_fetch_and_merge(args, repo_path, branch=None):
     if not args.enable_git:
         return