rename archive_org to archivedotorg, add BinaryWorker, fix config pass-through

Author: pirate

.claude/settings.local.json

@@ -1,6 +1,9 @@
 {
   "permissions": {
     "allow": [
+      "Read(**)",
+      "Glob(**)",
+      "Grep(**)",
       "Bash(python -m archivebox:*)",
       "Bash(ls:*)",
       "Bash(xargs:*)",
@@ -29,5 +32,19 @@
       "Bash(done)",
       "Bash(coverage erase:*)"
     ]
+  },
+  "hooks": {
+    "PreToolUse": [
+      {
+        "matcher": "Bash",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null); if [ -n \"$REPO_ROOT\" ] && [ \"$PWD\" != \"$REPO_ROOT\" ]; then echo \"ERROR: Not in repo root ($REPO_ROOT). Current dir: $PWD\" >&2; exit 1; fi",
+            "statusMessage": "Checking working directory..."
+          }
+        ]
+      }
+    ]
   }
 }

archivebox/base_models/models.py

@@ -128,4 +128,4 @@ def output_dir_str(self) -> str:
 
     @property
     def output_dir(self) -> Path:
-        raise NotImplementedError(f'{self.__class__.__name__} must implement output_dir property')
+        raise NotImplementedError(f"{self.__class__.__name__} must implement output_dir property")

archivebox/cli/archivebox_run.py

@@ -59,10 +59,11 @@ def process_stdin_records() -> int:
     """
     from django.utils import timezone
 
-    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_CRAWL, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
+    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_CRAWL, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT, TYPE_BINARY
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.core.models import Snapshot, ArchiveResult
     from archivebox.crawls.models import Crawl
+    from archivebox.machine.models import Binary
     from archivebox.workers.orchestrator import Orchestrator
 
     records = list(read_stdin())
@@ -137,6 +138,26 @@ def process_stdin_records() -> int:
                     output_records.append(archiveresult.to_json())
                     queued_count += 1
 
+            elif record_type == TYPE_BINARY:
+                # Binary records - create or update and queue for installation
+                if record_id:
+                    # Existing binary - re-queue
+                    try:
+                        binary = Binary.objects.get(id=record_id)
+                    except Binary.DoesNotExist:
+                        binary = Binary.from_json(record)
+                else:
+                    # New binary - create it
+                    binary = Binary.from_json(record)
+
+                if binary:
+                    binary.retry_at = timezone.now()
+                    if binary.status != Binary.StatusChoices.INSTALLED:
+                        binary.status = Binary.StatusChoices.QUEUED
+                    binary.save()
+                    output_records.append(binary.to_json())
+                    queued_count += 1
+
             else:
                 # Unknown type - pass through
                 output_records.append(record)
@@ -222,7 +243,8 @@ def run_snapshot_worker(snapshot_id: str) -> int:
 @click.option('--daemon', '-d', is_flag=True, help="Run forever (don't exit on idle)")
 @click.option('--crawl-id', help="Run orchestrator for specific crawl only")
 @click.option('--snapshot-id', help="Run worker for specific snapshot only")
-def main(daemon: bool, crawl_id: str, snapshot_id: str):
+@click.option('--binary-id', help="Run worker for specific binary only")
+def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str):
     """
     Process queued work.
 
@@ -231,11 +253,27 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str):
     - No args + TTY: Run orchestrator for all work
     - --crawl-id: Run orchestrator for that crawl only
     - --snapshot-id: Run worker for that snapshot only
+    - --binary-id: Run worker for that binary only
     """
     # Snapshot worker mode
     if snapshot_id:
         sys.exit(run_snapshot_worker(snapshot_id))
 
+    # Binary worker mode
+    if binary_id:
+        from archivebox.workers.worker import BinaryWorker
+        try:
+            worker = BinaryWorker(binary_id=binary_id, worker_id=0)
+            worker.runloop()
+            sys.exit(0)
+        except KeyboardInterrupt:
+            sys.exit(0)
+        except Exception as e:
+            rprint(f'[red]Worker error: {type(e).__name__}: {e}[/red]', file=sys.stderr)
+            import traceback
+            traceback.print_exc()
+            sys.exit(1)
+
     # Crawl worker mode
     if crawl_id:
         from archivebox.workers.worker import CrawlWorker

archivebox/config/configset.py

@@ -123,6 +123,7 @@ def get_config(
     user: Any = None,
     crawl: Any = None,
     snapshot: Any = None,
+    archiveresult: Any = None,
     machine: Any = None,
 ) -> Dict[str, Any]:
     """
@@ -145,11 +146,26 @@ def get_config(
         user: User object with config JSON field
         crawl: Crawl object with config JSON field
         snapshot: Snapshot object with config JSON field
+        archiveresult: ArchiveResult object (auto-fetches snapshot)
         machine: Machine object with config JSON field (defaults to Machine.current())
 
+    Note: Objects are auto-fetched from relationships if not provided:
+        - snapshot auto-fetched from archiveresult.snapshot
+        - crawl auto-fetched from snapshot.crawl
+        - user auto-fetched from crawl.created_by
+
     Returns:
         Merged config dict
     """
+    # Auto-fetch related objects from relationships
+    if snapshot is None and archiveresult and hasattr(archiveresult, "snapshot"):
+        snapshot = archiveresult.snapshot
+
+    if crawl is None and snapshot and hasattr(snapshot, "crawl"):
+        crawl = snapshot.crawl
+
+    if user is None and crawl and hasattr(crawl, "created_by"):
+        user = crawl.created_by
     from archivebox.config.constants import CONSTANTS
     from archivebox.config.common import (
         SHELL_CONFIG,
@@ -197,12 +213,18 @@ def get_config(
     if machine and hasattr(machine, "config") and machine.config:
         config.update(machine.config)
 
-    # Override with environment variables
+    # Override with environment variables (for keys that exist in config)
     for key in config:
         env_val = os.environ.get(key)
         if env_val is not None:
             config[key] = _parse_env_value(env_val, config.get(key))
 
+    # Also add NEW environment variables (not yet in config)
+    # This is important for worker subprocesses that receive config via Process.env
+    for key, value in os.environ.items():
+        if key.isupper() and key not in config:  # Only uppercase keys (config convention)
+            config[key] = _parse_env_value(value, None)
+
     # Also check plugin config aliases in environment
     try:
         from archivebox.hooks import discover_plugin_configs
@@ -335,7 +357,7 @@ def _parse_env_value(value: str, default: Any = None) -> Any:
     "title": 5,
     "favicon": 5,
     "headers": 5,
-    "archive_org": 2,
+    "archivedotorg": 2,
     "readability": 3,
     "mercury": 3,
     "git": 2,

archivebox/core/forms.py

@@ -147,7 +147,7 @@ def __init__(self, *args, **kwargs):
             'screenshot', 'seo', 'singlefile', 'ssl', 'staticfile', 'title'
         }
         archiving = {
-            'archive_org', 'favicon', 'forumdl', 'gallerydl', 'git',
+            'archivedotorg', 'favicon', 'forumdl', 'gallerydl', 'git',
             'htmltotext', 'media', 'mercury', 'papersdl', 'readability', 'wget'
         }
         parsing = {

archivebox/core/migrations/0007_archiveresult.py

@@ -120,7 +120,7 @@ class Migration(migrations.Migration):
                 ('output', models.CharField(max_length=512)),
                 ('start_ts', models.DateTimeField()),
                 ('end_ts', models.DateTimeField()),
-                ('extractor', models.CharField(choices=[('title', 'title'), ('favicon', 'favicon'), ('wget', 'wget'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('readability', 'readability'), ('mercury', 'mercury'), ('git', 'git'), ('media', 'media'), ('headers', 'headers'), ('archive_org', 'archive_org')], max_length=32)),
+                ('extractor', models.CharField(choices=[('title', 'title'), ('favicon', 'favicon'), ('wget', 'wget'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('readability', 'readability'), ('mercury', 'mercury'), ('git', 'git'), ('media', 'media'), ('headers', 'headers'), ('archivedotorg', 'archivedotorg')], max_length=32)),
                 ('snapshot', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.Snapshot')),
             ],
         ),

archivebox/core/migrations/0011_auto_20210216_1331.py

@@ -19,6 +19,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='archiveresult',
             name='extractor',
-            field=models.CharField(choices=[('title', 'title'), ('favicon', 'favicon'), ('headers', 'headers'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('wget', 'wget'), ('readability', 'readability'), ('mercury', 'mercury'), ('git', 'git'), ('media', 'media'), ('archive_org', 'archive_org')], max_length=32),
+            field=models.CharField(choices=[('title', 'title'), ('favicon', 'favicon'), ('headers', 'headers'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('wget', 'wget'), ('readability', 'readability'), ('mercury', 'mercury'), ('git', 'git'), ('media', 'media'), ('archivedotorg', 'archivedotorg')], max_length=32),
         ),
     ]

archivebox/core/migrations/0021_auto_20220914_0934.py

@@ -13,6 +13,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='archiveresult',
             name='extractor',
-            field=models.CharField(choices=[('favicon', 'favicon'), ('headers', 'headers'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('wget', 'wget'), ('title', 'title'), ('readability', 'readability'), ('mercury', 'mercury'), ('git', 'git'), ('media', 'media'), ('archive_org', 'archive_org')], max_length=32),
+            field=models.CharField(choices=[('favicon', 'favicon'), ('headers', 'headers'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('wget', 'wget'), ('title', 'title'), ('readability', 'readability'), ('mercury', 'mercury'), ('git', 'git'), ('media', 'media'), ('archivedotorg', 'archivedotorg')], max_length=32),
         ),
     ]

archivebox/core/migrations/0022_auto_20231023_2008.py

@@ -13,6 +13,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='archiveresult',
             name='extractor',
-            field=models.CharField(choices=[('favicon', 'favicon'), ('headers', 'headers'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('wget', 'wget'), ('title', 'title'), ('readability', 'readability'), ('mercury', 'mercury'), ('htmltotext', 'htmltotext'), ('git', 'git'), ('media', 'media'), ('archive_org', 'archive_org')], max_length=32),
+            field=models.CharField(choices=[('favicon', 'favicon'), ('headers', 'headers'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('wget', 'wget'), ('title', 'title'), ('readability', 'readability'), ('mercury', 'mercury'), ('htmltotext', 'htmltotext'), ('git', 'git'), ('media', 'media'), ('archivedotorg', 'archivedotorg')], max_length=32),
         ),
     ]

archivebox/core/models.py

@@ -1973,7 +1973,7 @@ def find_best_output_in_dir(dir_path: Path, plugin_name: str) -> Optional[str]:
         canonical = {
             'index_path': 'index.html',
             'google_favicon_path': FAVICON_PROVIDER.format(self.domain),
-            'archive_org_path': f'https://web.archive.org/web/{self.base_url}',
+            'archivedotorg_path': f'https://web.archive.org/web/{self.base_url}',
         }
 
         # Scan each ArchiveResult's output directory for the best file