Moved sessions to main function

Author: Joyakis

scripts/1-fetch/arxiv_fetch.py

@@ -333,9 +333,6 @@ def initialize_all_data_files(args):
     initialize_data_file(FILE_ARXIV_AUTHOR_BUCKET, HEADER_AUTHOR_BUCKET)
 
 
-session = shared.get_requests_session()
-
-
 def normalize_license_text(raw_text):
     """
     Convert raw license text to standardized CC license identifiers.

scripts/1-fetch/europeana_fetch.py

@@ -102,9 +102,6 @@ def parse_arguments():
     return args
 
 
-session = shared.get_requests_session(accept_header="application/json")
-
-
 def simplify_legal_tool(legal_tool):
     """Simplify license URLs into human-readable labels
 
@@ -422,7 +419,7 @@ def main():
             "EUROPEANA_API_KEY not found in environment variables", 1
         )
 
-    session = shared.get_requests_session()
+    session = shared.get_requests_session(accept_header="application/json")
 
     # Fetch facet lists once, including counts
     providers_full = get_facet_list(session, "DATA_PROVIDER")

scripts/1-fetch/github_fetch.py

@@ -78,13 +78,6 @@ def check_for_completion():
         pass  # File may not be found without --enable-save, etc.
 
 
-session = shared.get_requests_session(
-    headers={"accept": "application/vnd.github+json"},
-    auth_token=GH_TOKEN,
-    auth_prefix="Bearer",
-)
-
-
 def write_data(args, tool_data):
     if not args.enable_save:
         return args
@@ -148,7 +141,11 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     check_for_completion()
-    session = shared.get_requests_session()
+    session = shared.get_requests_session(
+        headers={"accept": "application/vnd.github+json"},
+        auth_token=GH_TOKEN,
+        auth_prefix="Bearer",
+    )
     tool_data = query_github(args, session)
     args = write_data(args, tool_data)
     args = shared.git_add_and_commit(

scripts/1-fetch/openverse_fetch.py

@@ -81,9 +81,6 @@ def parse_arguments():
     return args
 
 
-session = shared.get_requests_session(accept_header="application/json")
-
-
 def get_all_sources_and_licenses(session, media_type):
     """
     Fetch all available sources for a given media_type.
@@ -212,8 +209,8 @@ def write_data(args, data):
 
 def main():
     args = parse_arguments()
-    session = shared.get_requests_session()
     LOGGER.info("Starting Openverse Fetch Script...")
+    session = shared.get_requests_session(accept_header="application/json")
     records = query_openverse(session)
     write_data(args, records)
     LOGGER.info(f"Fetched {len(records)} unique Openverse records.")

scripts/1-fetch/wikipedia_fetch.py

@@ -63,9 +63,6 @@ def parse_arguments():
     return args
 
 
-session = shared.get_requests_session()
-
-
 def write_data(args, tool_data):
     if not args.enable_save:
         return args