Implement CLI resource export command

Closes #1000

Author: bkis

Tekst-API/tekst/__main__.py

@@ -1,13 +1,26 @@
 import asyncio
 
+from pathlib import Path
+from typing import get_args
+
 import click
 
+from beanie import PydanticObjectId
+from beanie.operators import Eq, In
+
 from tekst.config import get_config
 from tekst.db import init_odm, migrations
+from tekst.errors import TekstHTTPException
+from tekst.models.resource import (
+    ResourceBaseDocument,
+    ResourceExportFormat,
+    res_exp_fmt_info,
+)
 from tekst.openapi import generate_openapi_json
 from tekst.platform import bootstrap as app_bootstrap
 from tekst.platform import cleanup_task
 from tekst.resources import call_resource_precompute_hooks
+from tekst.routers.resources import export_resource_contents_task
 from tekst.search import create_indices_task
 
 
@@ -16,32 +29,97 @@
 """
 
 
-async def _prepare_odm() -> None:
-    await init_odm()
-
-
 async def _create_indices() -> None:
-    await _prepare_odm()
+    await init_odm()
     await create_indices_task()
 
 
 async def _refresh_precomputed_cache() -> None:
-    await _prepare_odm()
+    await init_odm()
     await call_resource_precompute_hooks()
 
 
 async def _cleanup() -> None:
-    await _prepare_odm()
+    await init_odm()
     await cleanup_task()
 
 
 async def _maintenance() -> None:
-    await _prepare_odm()
+    await init_odm()
     await create_indices_task()
     await call_resource_precompute_hooks()
     await cleanup_task()
 
 
+async def _export(
+    resource_ids: list[PydanticObjectId] | None,
+    *,
+    formats: list[ResourceExportFormat],
+    output_dir_path: Path,
+    quiet: bool = False,
+    delete: bool = False,
+) -> None:
+    # check output path
+    if not output_dir_path.exists():
+        output_dir_path.mkdir(parents=True)
+    if not output_dir_path.is_dir():
+        click.echo(f"Output directory {output_dir_path} is not a directory", err=True)
+        exit(1)
+    # delete all existing files in output directory
+    if delete:
+        if not quiet:
+            click.echo(f"Deleting all existing files in {output_dir_path} ...")
+        for child in output_dir_path.iterdir():
+            if child.is_file():
+                child.unlink()
+    # prepare system
+    await init_odm()
+    await call_resource_precompute_hooks()
+    # get IDs of resources to export
+    target_resources = await ResourceBaseDocument.find(
+        Eq(ResourceBaseDocument.public, True),
+        In(ResourceBaseDocument.id, resource_ids) if resource_ids else {},
+        with_children=True,
+    ).to_list()
+    # give feedback on resources that could not be found
+    for res in resource_ids or []:
+        if res not in [res.id for res in target_resources]:
+            click.echo(f"Resource ID {res} not found or not public", err=True)
+    # run exports
+    if not target_resources:
+        click.echo("No resources to export", err=True)
+        exit(1)
+    cfg = get_config()
+    for res in target_resources:
+        res_id_str = str(res.id)
+        for fmt in formats:
+            if not quiet:
+                click.echo(f"Exporting resource {res_id_str} as {fmt} ...")
+            try:
+                export_props = await export_resource_contents_task(
+                    user=None,
+                    cfg=cfg,
+                    resource_id=res.id,
+                    export_format=fmt,
+                )
+            except TekstHTTPException as e:
+                if e.detail.detail.key == "unsupportedExportFormat":
+                    click.echo(
+                        f"Resource {res_id_str} does not support export format {fmt}",
+                        err=True,
+                    )
+                    continue
+                else:
+                    raise
+            # move exported file to output directory
+            source_path = cfg.temp_files_dir / export_props["artifact"]
+            target_ext = res_exp_fmt_info[fmt]["extension"]
+            target_path = output_dir_path / f"{res_id_str}_{fmt}.{target_ext}"
+            source_path.replace(target_path)
+            if not quiet:
+                click.echo(f"Exported resource {res_id_str} as {str(target_path)}.")
+
+
 @click.command()
 def bootstrap():
     """Runs the Tekst initial bootstrap procedure"""
@@ -159,6 +237,69 @@ def schema(
         click.echo(schema_str)
 
 
+@click.command()
+@click.option(
+    "--resource",
+    "-r",
+    required=False,
+    default=None,
+    show_default=True,
+    multiple=True,
+    help="ID(s) of resource(s) to export, all if not set",
+)
+@click.option(
+    "--format",
+    "-f",
+    required=False,
+    default=get_args(ResourceExportFormat.__value__),
+    show_default=True,
+    multiple=True,
+    help="Format(s) to export, all if not set",
+)
+@click.option(
+    "--output",
+    "-o",
+    required=False,
+    default="/tmp/tekst_resource_export/",
+    show_default=True,
+    help="Output directory to write to",
+)
+@click.option(
+    "--quiet",
+    "-q",
+    is_flag=True,
+    default=False,
+    help="Don't output anything (except errors and warnings)",
+)
+@click.option(
+    "--delete",
+    "-d",
+    is_flag=True,
+    default=False,
+    help="Delete all existing files in the output directory",
+)
+def export(
+    resource: list[str] | None,
+    format: list[ResourceExportFormat],
+    output: str,
+    quiet: bool = False,
+    delete: bool = False,
+) -> None:
+    """
+    Exports the contents of the given resources (or all)
+    using the given formats (or all) to the given output directory
+    """
+    asyncio.run(
+        _export(
+            [PydanticObjectId(res_id) for res_id in resource] if resource else None,
+            formats=format,
+            output_dir_path=Path(output),
+            quiet=quiet,
+            delete=delete,
+        )
+    )
+
+
 @click.group()
 def cli():
     """Command line interface to the main functionalities of Tekst server"""
@@ -173,6 +314,7 @@ def cli():
 cli.add_command(maintenance)
 cli.add_command(migrate)
 cli.add_command(schema)
+cli.add_command(export)
 
 
 if __name__ == "__main__":

Tekst-API/tekst/models/resource.py

@@ -589,7 +589,7 @@ class ResourceCoverage(ModelBase):
     details: list[ParentCoverage]
 
 
-ResourceExportFormat = Literal["json", "tekst-json", "csv"]
+type ResourceExportFormat = Literal["json", "tekst-json", "csv"]
 
 res_exp_fmt_info = {
     "json": {
@@ -604,12 +604,4 @@ class ResourceCoverage(ModelBase):
         "extension": "csv",
         "mimetype": "text/csv",
     },
-    "txt": {
-        "extension": "txt",
-        "mimetype": "text/plain",
-    },
-    "html": {
-        "extension": "html",
-        "mimetype": "text/html",
-    },
 }

Tekst-API/tekst/routers/resources.py

@@ -995,13 +995,13 @@ async def import_resource_contents(
     )
 
 
-async def _export_resource_contents_task(
+async def export_resource_contents_task(
     user: OptionalUserDep,
     cfg: TekstConfig,
     resource_id: PydanticObjectId,
     export_format: ResourceExportFormat,
-    location_from_id: PydanticObjectId | None,
-    location_to_id: PydanticObjectId | None,
+    location_from_id: PydanticObjectId | None = None,
+    location_to_id: PydanticObjectId | None = None,
 ) -> dict[str, Any]:
     # check if user has permission to read this resource, if so, fetch from DB
     resource: ResourceBaseDocument = await ResourceBaseDocument.find_one(
@@ -1142,7 +1142,7 @@ async def export_resource_contents(
         raise errors.E_403_FORBIDDEN
     # create and return background task
     return await tasks.create_task(
-        _export_resource_contents_task,
+        export_resource_contents_task,
         tasks.TaskType.RESOURCE_EXPORT,
         user_id=user.id if user else None,
         target_id=user.id