simplify get_session and improve header handling

Author: Joyakis

scripts/1-fetch/arxiv_fetch.py

@@ -518,7 +518,7 @@ def query_arxiv(args):
     """
 
     LOGGER.info("Beginning to fetch results from ArXiv API")
-    session = shared.get_requests_session()
+    session = shared.get_session()
 
     results_per_iteration = 50
 

scripts/1-fetch/europeana_fetch.py

@@ -419,7 +419,7 @@ def main():
             "EUROPEANA_API_KEY not found in environment variables", 1
         )
 
-    session = shared.get_requests_session(accept_header="application/json")
+    session = shared.get_session(accept_header="application/json")
 
     # Fetch facet lists once, including counts
     providers_full = get_facet_list(session, "DATA_PROVIDER")

scripts/1-fetch/github_fetch.py

@@ -141,11 +141,12 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     check_for_completion()
-    session = shared.get_requests_session(
-        headers={"accept": "application/vnd.github+json"},
-        auth_token=GH_TOKEN,
-        auth_prefix="Bearer",
+    session = shared.get_session(
+        accept_header="application/vnd.github+json",
     )
+    if GH_TOKEN:
+        session.headers.update({"authorization": f"Bearer {GH_TOKEN}"})
+
     tool_data = query_github(args, session)
     args = write_data(args, tool_data)
     args = shared.git_add_and_commit(

scripts/1-fetch/openverse_fetch.py

@@ -210,7 +210,7 @@ def write_data(args, data):
 def main():
     args = parse_arguments()
     LOGGER.info("Starting Openverse Fetch Script...")
-    session = shared.get_requests_session(accept_header="application/json")
+    session = shared.get_session(accept_header="application/json")
     records = query_openverse(session)
     write_data(args, records)
     LOGGER.info(f"Fetched {len(records)} unique Openverse records.")

scripts/1-fetch/wikipedia_fetch.py

@@ -158,7 +158,7 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    tool_data = query_wikipedia_languages(shared.get_requests_session())
+    tool_data = query_wikipedia_languages(shared.get_session())
     args = write_data(args, tool_data)
     args = shared.git_add_and_commit(
         args,

scripts/shared.py

@@ -33,31 +33,22 @@ def __init__(self, message, exit_code=None):
         super().__init__(self.message)
 
 
-def get_requests_session(
-    accept_header: str | None = None,
-    auth_token: str | None = None,
-    mount_https: bool = True,
-) -> requests.Session:
-    """Create a reusable requests session with retry logic."""
+def get_session(accept_header=None):
+    """Create a reusable HTTP session with retry logic."""
     retry_strategy = Retry(
         total=5,
         backoff_factor=10,
         status_forcelist=STATUS_FORCELIST,
     )
 
     session = requests.Session()
-    session.mount("https://", HTTPAdapter(max_retries=retry_strategy))
 
     headers = {"User-Agent": USER_AGENT}
     if accept_header:
         headers["accept"] = accept_header
-    if auth_token:
-        headers["authorization"] = f"Bearer {auth_token}"
-    # Mount retry adapter for HTTPS
-    if mount_https:
-        session.mount("https://", HTTPAdapter(max_retries=retry_strategy))
-
     session.headers.update(headers)
+
+    session.mount("https://", HTTPAdapter(max_retries=retry_strategy))
     return session