wip

Author: pirate

Dockerfile

@@ -393,7 +393,7 @@ VOLUME "$DATA_DIR"
 EXPOSE 8000
 
 HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --silent 'http://localhost:8000/health/' | grep -q 'OK'
+    CMD curl --silent 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
 
 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
 CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"]

README.md

@@ -104,7 +104,8 @@ archivebox init --setup
 curl -fsSL 'https://get.archivebox.io' | bash
 </code></pre>
 <br/>
-<sub>Open <a href="http://localhost:8000"><code>http://localhost:8000</code></a> to see your server's Web UI ➡️</sub>
+<sub>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sub><br/>
+<sub>Set <code>LISTEN_HOST</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sub>
 </details>
 <br/>
 
@@ -469,6 +470,7 @@ For more discussion on managed and paid hosting options see here: <a href="https
 #### ➡️&nbsp; Next Steps
 
 - Import URLs from some of the supported [Input Formats](#input-formats) or view the supported [Output Formats](#output-formats)...
+- (Optional) Create a persona and import browser cookies to archive logged-in sites: `archivebox persona create --import=chrome personal`
 - Tweak your UI or archiving behavior [Configuration](#configuration), read about some of the [Caveats](#caveats), or [Troubleshoot](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
 - Read about the [Dependencies](#dependencies) used for archiving, the [Upgrading Process](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives), or the [Archive Layout](#archive-layout) on disk...
 - Or check out our full [Documentation](#documentation) or [Community Wiki](#internet-archiving-ecosystem)...
@@ -495,6 +497,11 @@ docker compose run archivebox help
 
 # equivalent: docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
 docker run -it -v $PWD:/data archivebox/archivebox help
+
+# optional: import your browser cookies into a persona for logged-in archiving
+archivebox persona create --import=chrome personal
+# supported: chrome/chromium/brave/edge (Chromium-based only)
+# re-running import merges/dedupes cookies.txt (by domain/path/name) but replaces chrome_user_data
 ```
 
 #### ArchiveBox Subcommands
@@ -587,7 +594,8 @@ docker run -v $PWD:/data -it archivebox/archivebox archivebox manage createsuper
 docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
 </code></pre>
 
-<sup>Open <a href="http://localhost:8000"><code>http://localhost:8000</code></a> to see your server's Web UI ➡️</sup>
+<sup>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sup><br/>
+<sup>Set <code>LISTEN_HOST</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sup>
 <br/><br/>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage">Usage: Web UI</a> wiki. ➡️</i>
 <br/><br/>

archivebox/api/auth.py

@@ -127,12 +127,25 @@ class UsernameAndPasswordAuth(UserPassAuthCheck, HttpBasicAuth):
     """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
     pass
 
+class DjangoSessionAuth:
+    """Allow authenticating with existing Django session cookies (same-origin only)."""
+    def __call__(self, request: HttpRequest) -> Optional[AbstractBaseUser]:
+        return self.authenticate(request)
+
+    def authenticate(self, request: HttpRequest, **kwargs) -> Optional[AbstractBaseUser]:
+        user = getattr(request, 'user', None)
+        if user and user.is_authenticated:
+            request._api_auth_method = self.__class__.__name__
+            if not user.is_superuser:
+                raise HttpError(403, 'Valid session but User does not have permission (make sure user.is_superuser=True)')
+            return cast(AbstractBaseUser, user)
+        return None
+
 ### Enabled Auth Methods
 
 API_AUTH_METHODS = [
     HeaderTokenAuth(),
     BearerTokenAuth(),
     QueryParamTokenAuth(), 
     # django_auth_superuser,       # django admin cookie auth, not secure to use with csrf=False
-    UsernameAndPasswordAuth(),
 ]

archivebox/api/middleware.py

@@ -0,0 +1,34 @@
+__package__ = 'archivebox.api'
+
+from django.http import HttpResponse
+
+
+class ApiCorsMiddleware:
+    """Attach permissive CORS headers for API routes (token-based auth)."""
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        if request.path.startswith('/api/'):
+            if request.method == 'OPTIONS' and request.META.get('HTTP_ACCESS_CONTROL_REQUEST_METHOD'):
+                response = HttpResponse(status=204)
+                return self._add_cors_headers(request, response)
+
+            response = self.get_response(request)
+            return self._add_cors_headers(request, response)
+
+        return self.get_response(request)
+
+    def _add_cors_headers(self, request, response):
+        origin = request.META.get('HTTP_ORIGIN')
+        if not origin:
+            return response
+
+        response['Access-Control-Allow-Origin'] = '*'
+        response['Access-Control-Allow-Methods'] = 'GET, POST, PUT, PATCH, DELETE, OPTIONS'
+        response['Access-Control-Allow-Headers'] = (
+            'Authorization, X-ArchiveBox-API-Key, Content-Type, X-CSRFToken'
+        )
+        response['Access-Control-Max-Age'] = '600'
+        return response

archivebox/api/v1_core.py

@@ -188,6 +188,11 @@ def resolve_archiveresults(obj, context):
         return ArchiveResult.objects.none()
 
 
+class SnapshotUpdateSchema(Schema):
+    status: str | None = None
+    retry_at: datetime | None = None
+
+
 class SnapshotFilterSchema(FilterSchema):
     id: Optional[str] = Field(None, q=['id__icontains', 'timestamp__startswith'])
     created_by_id: str = Field(None, q='crawl__created_by_id')
@@ -225,6 +230,31 @@ def get_snapshot(request, snapshot_id: str, with_archiveresults: bool = True):
         return Snapshot.objects.get(Q(id__icontains=snapshot_id))
 
 
+@router.patch("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="patch_snapshot")
+def patch_snapshot(request, snapshot_id: str, data: SnapshotUpdateSchema):
+    """Update a snapshot (e.g., set status=sealed to cancel queued work)."""
+    try:
+        snapshot = Snapshot.objects.get(Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
+    except Snapshot.DoesNotExist:
+        snapshot = Snapshot.objects.get(Q(id__icontains=snapshot_id))
+
+    payload = data.dict(exclude_unset=True)
+
+    if 'status' in payload:
+        if payload['status'] not in Snapshot.StatusChoices.values:
+            raise HttpError(400, f'Invalid status: {payload["status"]}')
+        snapshot.status = payload['status']
+        if snapshot.status == Snapshot.StatusChoices.SEALED and 'retry_at' not in payload:
+            snapshot.retry_at = None
+
+    if 'retry_at' in payload:
+        snapshot.retry_at = payload['retry_at']
+
+    snapshot.save(update_fields=['status', 'retry_at', 'modified_at'])
+    request.with_archiveresults = False
+    return snapshot
+
+
 ### Tag #########################################################################
 
 class TagSchema(Schema):

archivebox/api/v1_crawls.py

@@ -3,11 +3,13 @@
 from uuid import UUID
 from typing import List
 from datetime import datetime
+from django.utils import timezone
 
 from django.db.models import Q
 from django.contrib.auth import get_user_model
 
 from ninja import Router, Schema
+from ninja.errors import HttpError
 
 from archivebox.core.models import Snapshot
 from archivebox.crawls.models import Crawl
@@ -54,6 +56,11 @@ def resolve_snapshots(obj, context):
         return Snapshot.objects.none()
 
 
+class CrawlUpdateSchema(Schema):
+    status: str | None = None
+    retry_at: datetime | None = None
+
+
 @router.get("/crawls", response=List[CrawlSchema], url_name="get_crawls")
 def get_crawls(request):
     return Crawl.objects.all().distinct()
@@ -79,3 +86,32 @@ def get_crawl(request, crawl_id: str, as_rss: bool=False, with_snapshots: bool=F
 
     return crawl
 
+
+@router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
+def patch_crawl(request, crawl_id: str, data: CrawlUpdateSchema):
+    """Update a crawl (e.g., set status=sealed to cancel queued work)."""
+    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    payload = data.dict(exclude_unset=True)
+
+    if 'status' in payload:
+        if payload['status'] not in Crawl.StatusChoices.values:
+            raise HttpError(400, f'Invalid status: {payload["status"]}')
+        crawl.status = payload['status']
+        if crawl.status == Crawl.StatusChoices.SEALED and 'retry_at' not in payload:
+            crawl.retry_at = None
+
+    if 'retry_at' in payload:
+        crawl.retry_at = payload['retry_at']
+
+    crawl.save(update_fields=['status', 'retry_at', 'modified_at'])
+
+    if payload.get('status') == Crawl.StatusChoices.SEALED:
+        Snapshot.objects.filter(
+            crawl=crawl,
+            status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED],
+        ).update(
+            status=Snapshot.StatusChoices.SEALED,
+            retry_at=None,
+            modified_at=timezone.now(),
+        )
+    return crawl