Merge branch 'main' into license-classify-modeling

Author: TimidRobot

.github/workflows/1-fetch.yml

@@ -0,0 +1,69 @@
+name: Fetch Data
+
+on:
+  schedule:
+    # Normal schedule
+    # # at 01:15 on all days in first month of each quarter
+    # - cron: '15 1 * 1,4,7,10 *'
+    # # at 01:15 on days 1-14 in second month of each quarter
+    # - cron: '15 1 1-14 2,5,8,11 *'
+    # Temp schedule
+    # at 01:15 on all days in all months
+    - cron: '15 1 * * *'
+
+  workflow_dispatch:
+
+jobs:
+  fetch:
+    runs-on: ubuntu-latest
+
+    steps:
+
+      # CC Technology team members:
+      #   See cc-quantifying-bot GitHub entry in Bitwarden for information on
+      #   BOT_ secrets
+      - name: Configure git
+        run: |
+          git config --global init.defaultBranch main
+          git config --global user.name "${{ secrets.BOT_NAME }}"
+          git config --global user.email "${{ secrets.BOT_EMAIL }}"
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          # Default fetch-depth is 1, however that value results in errors
+          # when GitPython attempts to push changes:
+          # "failed to push some refs"
+          fetch-depth: 0
+          token: ${{ secrets.BOT_TOKEN }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install Python dependencies
+        run: |
+          pip install --upgrade pip pipenv
+
+      - name: Sync Python modules
+        run: |
+          pipenv sync --system
+
+      # CC Technology team members:
+      #   See cc-quantifying-bot Google Workspace  entry in Bitwarden for
+      #   information on GCS_ secrets
+      - name: Fetch from Google Custom Search (GCS)
+        run: |
+          ./scripts/1-fetch/gcs_fetch.py \
+            --limit=100 --enable-save --enable-git
+        env:
+          GCS_DEVELOPER_KEY: ${{ secrets.GCS_DEVELOPER_KEY }}
+          GCS_CX: ${{ secrets.GCS_CX }}
+
+      - name: Fetch from GitHub
+        run: |
+          ./scripts/1-fetch/github_fetch.py \
+            --enable-save --enable-git
+        env:
+          GH_TOKEN: ${{ secrets.BOT_TOKEN }}

.github/workflows/2-process.yml.disabled

@@ -0,0 +1,46 @@
+name: Process Data
+
+on:
+  schedule:
+    # at 02:15 on days 15-28 in second month of each quarter
+    - cron: '15 2 15-28 2,5,8,11 *'
+  workflow_dispatch:
+
+jobs:
+  process:
+    runs-on: ubuntu-latest
+
+    # CC Technology team members:
+    #   See cc-quantifying-bot GitHub entry in Bitwarden for information on
+    #   BOT_ secrets
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.BOT_TOKEN }}
+
+      - name: Configure git
+        run: |
+          git config user.name "${{ secrets.BOT_NAME }}"
+          git config user.email "${{ secrets.BOT_EMAIL }}"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install pipenv
+          pipenv sync --system
+
+      - name: Set PYTHONPATH
+        run: echo "PYTHONPATH=./scripts" >> $GITHUB_ENV
+
+      - name: Run process script
+        run: |
+          # ADD SCRIPT INVOCATION HERE
+
+# vim: ft=yaml

.github/workflows/3-report.yml.disabled

@@ -0,0 +1,46 @@
+name: Generate Report
+
+on:
+  schedule:
+    # at 03:15 on all days in third month of each quarter
+    - cron: '15 3 * 3,6,9,12 *'
+  workflow_dispatch:
+
+jobs:
+  generate-report:
+    runs-on: ubuntu-latest
+
+    # CC Technology team members:
+    #   See cc-quantifying-bot GitHub entry in Bitwarden for information on
+    #   BOT_ secrets
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.BOT_TOKEN }}
+
+      - name: Configure git
+        run: |
+          git config user.name "${{ secrets.BOT_NAME }}"
+          git config user.email "${{ secrets.BOT_EMAIL }}"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install pipenv
+          pipenv sync --system
+
+      - name: Set PYTHONPATH
+        run: echo "PYTHONPATH=./scripts" >> $GITHUB_ENV
+
+      - name: Run report script
+        run: |
+          # ADD SCRIPT INVOCATION HERE
+
+# vim: ft=yaml

.gitignore

@@ -144,3 +144,6 @@ Thumbs.db
 # secrets
 .env
 query_secrets.py
+
+# backup files
+*.bak

Pipfile

@@ -5,17 +5,22 @@ name = "pypi"
 
 [packages]
 flickrapi = "*"
-internetarchive = "*"
+GitPython = "*"
+google-api-python-client = "*"
+h11 = ">=0.16.0"  # Ensure dependency is secure
+internetarchive = ">=5.5.1"
 jupyterlab = ">=3.6.7"
 matplotlib = "*"
 numpy = "*"
 pandas = "*"
 plotly = "*"
+pillow = ">=11.3.0"  # Ensure dependency is secure
 Pyarrow = "*"
+Pygments = "*"
 python-dotenv = "*"
 requests = ">=2.31.0"
 seaborn = "*"
-urllib3 = ">=1.26.18"
+urllib3 = ">=2.5.0"
 wordcloud = "*"
 
 [dev-packages]
@@ -27,3 +32,9 @@ pre-commit = "*"
 
 [requires]
 python_version = "3.11"
+
+[scripts]
+gcs_fetched = "./scripts/1-fetch/gcs_fetched.py"
+flickr_fetched = "./scripts/1-fetch/flickr_fetched.py"
+gcs_processed = "./scripts/2-process/gcs_processed.py"
+gcs_reports = "./scripts/3-report/gcs_reports.py"