newsletter cleanup test

Author: pubino

.github/workflows/process-newsletters.yml

@@ -34,6 +34,12 @@ jobs:
         run: |
           pip install beautifulsoup4 pyyaml git-filter-repo
 
+      - name: Install AI dependencies
+        if: env.AI_SANDBOX_KEY != ''
+        env:
+          AI_SANDBOX_KEY: ${{ secrets.AI_SANDBOX_KEY }}
+        run: pip install portkey-ai
+
       - name: Check for files to process
         id: check_files
         run: |
@@ -55,6 +61,22 @@ jobs:
         run: |
           python scripts/process_inbox.py
 
+      - name: AI post-process newsletters
+        if: steps.check_files.outputs.has_files == 'true' && env.AI_SANDBOX_KEY != ''
+        env:
+          AI_SANDBOX_KEY: ${{ secrets.AI_SANDBOX_KEY }}
+        run: |
+          # Find newly added newsletter files
+          FILES=$(git diff --name-only --diff-filter=A -- '_newsletters/*/index.md' 2>/dev/null || true)
+          if [ -n "$FILES" ]; then
+            echo "AI post-processing $(echo "$FILES" | wc -l | tr -d ' ') newsletter(s)..."
+            echo "$FILES" | while read -r f; do
+              python scripts/ai_post_process.py "$f" --verbose || echo "WARNING: AI post-processing failed for $f"
+            done
+          else
+            echo "No new newsletters to post-process"
+          fi
+
       - name: Commit processed newsletters
         if: steps.check_files.outputs.has_files == 'true'
         run: |

.github/workflows/receive-newsletter.yml

@@ -28,6 +28,12 @@ jobs:
       - name: Install dependencies
         run: pip install beautifulsoup4
 
+      - name: Install AI dependencies
+        if: env.AI_SANDBOX_KEY != ''
+        env:
+          AI_SANDBOX_KEY: ${{ secrets.AI_SANDBOX_KEY }}
+        run: pip install portkey-ai
+
       - name: Check for duplicate
         id: dedup
         env:
@@ -74,6 +80,17 @@ jobs:
           ensure_archive_pages(date)
           PYEOF
 
+      - name: AI post-process newsletter
+        if: steps.dedup.outputs.is_duplicate == 'false' && env.AI_SANDBOX_KEY != ''
+        env:
+          AI_SANDBOX_KEY: ${{ secrets.AI_SANDBOX_KEY }}
+          SLUG: ${{ github.event.client_payload.slug }}
+        run: |
+          FILEPATH="_newsletters/${SLUG}/index.md"
+          if [ -f "$FILEPATH" ]; then
+            python scripts/ai_post_process.py "$FILEPATH" --verbose || echo "WARNING: AI post-processing failed"
+          fi
+
       - name: Commit and push
         if: steps.dedup.outputs.is_duplicate == 'false'
         run: |

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help serve-local serve-preview serve-prod build-preview build-prod clean docker-local docker-preview docker-prod backfill backfill-dry-run
+.PHONY: help serve-local serve-preview serve-prod build-preview build-prod clean docker-local docker-preview docker-prod backfill backfill-dry-run ai-cleanup ai-cleanup-dry-run
 
 JEKYLL?=bundle exec jekyll
 PYTHON?=.venv/bin/python
@@ -53,3 +53,10 @@ backfill:
 
 backfill-dry-run:
 	$(PYTHON) scripts/backfill_newsletters.py --dir backfill/ --dry-run
+
+# AI post-processing (requires AI_SANDBOX_KEY env var and portkey-ai)
+ai-cleanup:
+	$(PYTHON) scripts/ai_post_process.py _newsletters/*/index.md
+
+ai-cleanup-dry-run:
+	$(PYTHON) scripts/ai_post_process.py _newsletters/*/index.md --dry-run