Merge branch 'main' into Wikipedia/processing

Author: TimidRobot

.cc-metadata.yml

@@ -6,5 +6,3 @@ english_name: Quantifying the Commons
 technologies: Python
 # Whether this repository should be featured on the CC Open Source site
 featured: true
-# Slack channel name
-slack: "cc-dev-quantifying"

.github/workflows/test_scripts_help.yml

@@ -0,0 +1,34 @@
+name: Test scripts' help
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  job:
+    runs-on: ubuntu-latest
+
+    steps:
+
+    # https://github.com/actions/setup-python
+    - name: Install Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+
+    - name: Install pipenv
+      run: |
+        pip install --upgrade pip
+        pip install pipenv
+
+    # https://github.com/actions/checkout
+    - name: Checkout quantifying
+      uses: actions/checkout@v4
+
+    - name: Install Python dependencies
+      run: |
+        pipenv sync --system
+
+    - name: Test scripts' help
+      run: |
+        ./dev/test_scripts_help.sh

dev/test_scripts_help.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+#
+# Ensure each script can display help message to ensure basic execution.
+#
+# This script must be run from within the pipenv shell or a properly configured
+# environment. For example:
+#
+#   1. Using pipenv run
+#       pipenv run ./dev/test_scripts_help.sh
+#
+#   2. Using pipenv shell
+#       pipenv shell
+#       ./dev/test_scripts_help.sh
+#
+#   3. A properly configured environment
+#      (see .github/workflows/test_scripts_help.yml)
+#
+#### SETUP ####################################################################
+
+set -o errexit
+set -o errtrace
+set -o nounset
+
+# shellcheck disable=SC2154
+trap '_es=${?};
+    printf "${0}: line ${LINENO}: \"${BASH_COMMAND}\"";
+    printf " exited with a status of ${_es}\n";
+    exit ${_es}' ERR
+
+DIR_REPO="$(cd -P -- "${0%/*}/.." && pwd -P)"
+EXIT_STATUS=0
+
+#### FUNCTIONS ################################################################
+
+test_help() {
+    local _es _script
+    for _script in $(find scripts/?-* -type f -name '*.py' | sort)
+    do
+        _es=0
+        ./"${_script}" --help &>/dev/null || _es=${?}
+        if (( _es == 0 ))
+        then
+            echo "✅ ${_script}"
+        else
+            echo "❌ ${_script}"
+            EXIT_STATUS=${_es}
+        fi
+    done
+}
+
+#### MAIN #####################################################################
+
+cd "${DIR_REPO}"
+test_help
+echo "exit status: ${EXIT_STATUS}"
+exit ${EXIT_STATUS}

scripts/1-fetch/arxiv_fetch.py

@@ -22,8 +22,6 @@
 from pygments import highlight
 from pygments.formatters import TerminalFormatter
 from pygments.lexers import PythonTracebackLexer
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
 
 # Add parent directory so shared can be imported
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -36,7 +34,7 @@
 
 # Constants
 # API Configuration
-BASE_URL = "http://export.arxiv.org/api/query?"
+BASE_URL = "https://export.arxiv.org/api/query?"
 DEFAULT_FETCH_LIMIT = 800  # Default total papers to fetch
 
 # CSV Headers
@@ -335,19 +333,6 @@ def initialize_all_data_files(args):
     initialize_data_file(FILE_ARXIV_AUTHOR_BUCKET, HEADER_AUTHOR_BUCKET)
 
 
-def get_requests_session():
-    """Create request session with retry logic"""
-    retry_strategy = Retry(
-        total=5,
-        backoff_factor=10,
-        status_forcelist=shared.STATUS_FORCELIST,
-    )
-    session = requests.Session()
-    session.headers.update({"User-Agent": shared.USER_AGENT})
-    session.mount("https://", HTTPAdapter(max_retries=retry_strategy))
-    return session
-
-
 def normalize_license_text(raw_text):
     """
     Convert raw license text to standardized CC license identifiers.
@@ -533,7 +518,7 @@ def query_arxiv(args):
     """
 
     LOGGER.info("Beginning to fetch results from ArXiv API")
-    session = get_requests_session()
+    session = shared.get_session()
 
     results_per_iteration = 50
 

scripts/1-fetch/europeana_fetch.py

@@ -23,7 +23,6 @@
 from pygments import highlight
 from pygments.formatters import TerminalFormatter
 from pygments.lexers import PythonTracebackLexer
-from requests.adapters import HTTPAdapter, Retry
 
 # Add parent directory for shared imports
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -103,19 +102,6 @@ def parse_arguments():
     return args
 
 
-def get_requests_session():
-    """Create a requests session with retry."""
-    max_retries = Retry(
-        total=5, backoff_factor=10, status_forcelist=shared.STATUS_FORCELIST
-    )
-    session = requests.Session()
-    session.mount("https://", HTTPAdapter(max_retries=max_retries))
-    session.headers.update(
-        {"accept": "application/json", "User-Agent": shared.USER_AGENT}
-    )
-    return session
-
-
 def simplify_legal_tool(legal_tool):
     """Simplify license URLs into human-readable labels
 
@@ -433,7 +419,7 @@ def main():
             "EUROPEANA_API_KEY not found in environment variables", 1
         )
 
-    session = get_requests_session()
+    session = shared.get_session(accept_header="application/json")
 
     # Fetch facet lists once, including counts
     providers_full = get_facet_list(session, "DATA_PROVIDER")

scripts/1-fetch/github_fetch.py

@@ -17,8 +17,6 @@
 from pygments import highlight
 from pygments.formatters import TerminalFormatter
 from pygments.lexers import PythonTracebackLexer
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
 
 # Add parent directory so shared can be imported
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -80,25 +78,6 @@ def check_for_completion():
         pass  # File may not be found without --enable-save, etc.
 
 
-def get_requests_session():
-    max_retries = Retry(
-        total=5,
-        backoff_factor=10,
-        status_forcelist=shared.STATUS_FORCELIST,
-    )
-    session = requests.Session()
-    session.mount("https://", HTTPAdapter(max_retries=max_retries))
-    headers = {
-        "accept": "application/vnd.github+json",
-        "User-Agent": shared.USER_AGENT,
-    }
-    if GH_TOKEN:
-        headers["authorization"] = f"Bearer {GH_TOKEN}"
-    session.headers.update(headers)
-
-    return session
-
-
 def write_data(args, tool_data):
     if not args.enable_save:
         return args
@@ -162,7 +141,12 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     check_for_completion()
-    session = get_requests_session()
+    session = shared.get_session(
+        accept_header="application/vnd.github+json",
+    )
+    if GH_TOKEN:
+        session.headers.update({"authorization": f"Bearer {GH_TOKEN}"})
+
     tool_data = query_github(args, session)
     args = write_data(args, tool_data)
     args = shared.git_add_and_commit(

scripts/1-fetch/openverse_fetch.py

@@ -25,8 +25,6 @@
 from pygments import highlight
 from pygments.formatters import TerminalFormatter
 from pygments.lexers import PythonTracebackLexer
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
 
 # Add parent directory so shared can be imported
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -83,20 +81,6 @@ def parse_arguments():
     return args
 
 
-def get_requests_session():
-    max_retries = Retry(
-        total=5,
-        backoff_factor=10,
-        status_forcelist=shared.STATUS_FORCELIST,
-    )
-    session = requests.Session()
-    session.mount("https://", HTTPAdapter(max_retries=max_retries))
-    session.headers.update(
-        {"accept": "application/json", "User-Agent": shared.USER_AGENT}
-    )
-    return session
-
-
 def get_all_sources_and_licenses(session, media_type):
     """
     Fetch all available sources for a given media_type.
@@ -225,8 +209,8 @@ def write_data(args, data):
 
 def main():
     args = parse_arguments()
-    session = get_requests_session()
     LOGGER.info("Starting Openverse Fetch Script...")
+    session = shared.get_session(accept_header="application/json")
     records = query_openverse(session)
     write_data(args, records)
     LOGGER.info(f"Fetched {len(records)} unique Openverse records.")

scripts/1-fetch/wikipedia_fetch.py

@@ -13,12 +13,9 @@
 from operator import itemgetter
 
 # Third-party
-import requests
 from pygments import highlight
 from pygments.formatters import TerminalFormatter
 from pygments.lexers import PythonTracebackLexer
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
 
 # Add parent directory so shared can be imported
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -66,18 +63,6 @@ def parse_arguments():
     return args
 
 
-def get_requests_session():
-    max_retries = Retry(
-        total=5,
-        backoff_factor=10,
-        status_forcelist=shared.STATUS_FORCELIST,
-    )
-    session = requests.Session()
-    session.mount("https://", HTTPAdapter(max_retries=max_retries))
-    session.headers.update({"User-Agent": shared.USER_AGENT})
-    return session
-
-
 def write_data(args, tool_data):
     if not args.enable_save:
         return args
@@ -173,7 +158,8 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    tool_data = query_wikipedia_languages(get_requests_session())
+    session = shared.get_session()
+    tool_data = query_wikipedia_languages(session)
     args = write_data(args, tool_data)
     args = shared.git_add_and_commit(
         args,

scripts/shared.py

@@ -2,11 +2,15 @@
 import logging
 import os
 import sys
+from collections import OrderedDict
 from datetime import datetime, timezone
 
 # Third-party
 from git import InvalidGitRepositoryError, NoSuchPathError, Repo
 from pandas import PeriodIndex
+from requests import Session
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
 
 # Constants
 STATUS_FORCELIST = [
@@ -31,6 +35,37 @@ def __init__(self, message, exit_code=None):
         super().__init__(self.message)
 
 
+def get_session(accept_header=None, session=None):
+    """
+    Create or configure a reusable HTTPS session with retry logic and
+    appropriate headers.
+    """
+    if session is None:
+        session = Session()
+
+    # Purge default and custom session connection adapters
+    # (With only a https:// adapter, below, unencrypted requests will fail.)
+    session.adapters = OrderedDict()
+
+    # Try again after 0s, 6s, 12s, 24s, 48s (total 90s) for the specified HTTP
+    # error codes (STATUS_FORCELIST)
+    retry_strategy = Retry(
+        total=5,
+        backoff_factor=3,
+        status_forcelist=STATUS_FORCELIST,
+        allowed_methods=["GET", "POST"],
+        raise_on_status=False,
+    )
+    session.mount("https://", HTTPAdapter(max_retries=retry_strategy))
+
+    headers = {"User-Agent": USER_AGENT}
+    if accept_header:
+        headers["accept"] = accept_header
+    session.headers.update(headers)
+
+    return session
+
+
 def git_fetch_and_merge(args, repo_path, branch=None):
     if not args.enable_git:
         return

sources.md

@@ -121,6 +121,32 @@ and access towards related query data using a programmable search engine.
 - Data available through JSON format
 
 
+## Openverse
+
+**Description:** Openverse is a search engine for openly licensed media,
+including images and audio. It provides access to over 700 million works from
+more than 20 sources, all of which are under Creative Commons licenses or in the
+public domain. The API allows querying for media by source, license type, and
+other parameters. Because anonymous Openverse API access returns a maximum of
+~240 result count per source-license combination, the `openverse_fetch.py`
+script currently provides approximate counts. It does not include pagination or
+license_version breakdown.
+
+**API documentation link:**
+- [Openverse API Documentation](https://api.openverse.org/v1/)
+- [Openverse API Reference](https://wordpress.org/openverse/api/)
+- [Base URL](https://api.openverse.org/v1)
+- [Openverse Frontend](https://openverse.org/)
+
+**API information:**
+- No API key required for basic access
+- Query limit: Rate-limited to prevent abuse (anonymous access provides ~240 results per source-license combination)
+- Data available through JSON format
+- Supports filtering by source, license, media type (images, audio)
+- Media types: `images`, `audio`
+- Supported licenses: `by`, `by-nc`, `by-nc-nd`, `by-nc-sa`, `by-nd`, `by-sa`, `cc0`, `nc-sampling+`, `pdm`, `sampling+`
+
+
 ## Wikipedia
 
 **Description:** The Wikipedia API allows users to query statistics of pages,