Implement migrate command (#443)

* Implement migrate command

* Update changelog

* Add recursive and old version of planet-disaster

* Raise if no-op

* Move changelog entry

* Update src/stactools/cli/commands/migrate.py

---------

Co-authored-by: Pete Gadomski <[email protected]>

Author: jsignell

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Add `stac migrate` command ([#443](https://github.com/stac-utils/stactools/pull/443))
+
 ### Fixed
 
 - `reproject_shape` without a precision ([#454](https://github.com/stac-utils/stactools/pull/454))

src/stactools/cli/__init__.py

@@ -37,15 +37,13 @@ def register_plugin(registry: "Registry") -> None:
     registry.register_subcommand(layout.create_layout_command)
     registry.register_subcommand(lint.create_lint_command)
     registry.register_subcommand(merge.create_merge_command)
+    registry.register_subcommand(migrate.create_migrate_command)
     registry.register_subcommand(summary.create_summary_command)
     registry.register_subcommand(validate.create_validate_command)
     registry.register_subcommand(version.create_version_command)
     registry.register_subcommand(update_extent.create_update_extent_command)
     registry.register_subcommand(update_geometry.create_update_geometry_command)
 
-    # TODO
-    # registry.register_subcommand(migrate.create_migrate_command)
-
 
 from stactools.cli.registry import Registry
 

src/stactools/cli/commands/migrate.py

@@ -1,15 +1,54 @@
 import click
+import pystac
+from stactools.core import migrate_object
 
 
-def migrate(option: click.Option) -> None:
-    # TODO
-    print(option)
+def _migrate(
+    href: str, save: bool = False, recursive: bool = False, show_diff: bool = True
+) -> pystac.STACObject:
+    if save is False and show_diff is False:
+        raise click.BadArgumentUsage(
+            "It is only valid to use 'hide-diff' when 'save' is enabled "
+            "otherwise there would be no output."
+        )
+
+    stac_object = pystac.read_file(href)
+    if recursive and not isinstance(stac_object, (pystac.Catalog, pystac.Collection)):
+        raise click.BadArgumentUsage(
+            "'recursive' is only a valid option for "
+            "pystac.Catalogs and pystac.Collections"
+        )
+    return migrate_object(
+        stac_object, save=save, recursive=recursive, show_diff=show_diff
+    )
 
 
 def create_migrate_command(cli: click.Group) -> click.Command:
-    @cli.command("migrate", short_help="Migrate a STAC catalog (TODO)")
-    @click.option("--option", "-o", default=1, help="A test option")
-    def migrate_command(option: click.Option) -> None:
-        migrate(option)
+    @cli.command("migrate", short_help="Migrate a STAC object to the latest version")
+    @click.argument("href")
+    @click.option(
+        "-s",
+        "--save",
+        is_flag=True,
+        help="Save migrated STAC object in original location.",
+    )
+    @click.option(
+        "-r",
+        "--recursive",
+        is_flag=True,
+        help="Recurse through all child objects and migrate them as well.",
+    )
+    @click.option(
+        "--show-diff/--hide-diff",
+        default=True,
+        help=(
+            "Whether to dump diff between original and migrated object to stdout. "
+            "Defaults to --show-diff. "
+        ),
+    )
+    def migrate_command(
+        href: str, save: bool, recursive: bool, show_diff: bool
+    ) -> None:
+        _migrate(href, save=save, recursive=recursive, show_diff=show_diff)
 
     return migrate_command

src/stactools/core/__init__.py

@@ -11,6 +11,7 @@
 from stactools.core.io import use_fsspec
 from stactools.core.layout import layout_catalog
 from stactools.core.merge import merge_all_items, merge_items
+from stactools.core.migrate import migrate_object
 
 __all__ = [
     "add_item",
@@ -21,6 +22,7 @@
     "layout_catalog",
     "merge_all_items",
     "merge_items",
+    "migrate_object",
     "move_asset_file",
     "move_asset_file_to_item",
     "move_assets",

src/stactools/core/migrate.py

@@ -0,0 +1,65 @@
+import difflib
+import json
+from itertools import chain
+
+import pystac
+from pystac.stac_io import DefaultStacIO
+
+
+def _print_diff(stac_object: pystac.STACObject) -> None:
+    href = stac_object.get_self_href()
+    if href is None:
+        raise ValueError(f"Could not determine diff for {stac_object}, missing href")
+    input = DefaultStacIO().read_text(href)
+    output = json.dumps(stac_object.to_dict(), indent=2)
+    for diffs in difflib.unified_diff(
+        input.splitlines(), output.splitlines(), fromfile=f"a{href}", tofile=f"b{href}"
+    ):
+        print(diffs)
+
+
+def migrate_object(
+    stac_object: pystac.STACObject,
+    save: bool = False,
+    recursive: bool = False,
+    show_diff: bool = True,
+) -> pystac.STACObject:
+    """Migrate a STAC object and all its extensions to the latest version of STAC
+
+    Note:
+        Migrating a STAC object will set the keys to be in the standard order.
+        This might result in a diff even for a valid and up-to-date STAC object.
+
+    Args:
+        stac_object : STAC object to migrate
+        save : Whether to save the object back to its original location.
+            Defaults to False.
+        recursive : Whether to recurse through all the child object and migrate
+            them as well. Defaults to False.
+        show_diff : Whether to print the diff between the original and new STAC
+            object. Defaults to True.
+    Returns:
+        STACObject : The migrated object - modified inplace.
+    """
+    # migration happens implicitly on load
+    if show_diff:
+        _print_diff(stac_object)
+
+    if recursive:
+        if not isinstance(stac_object, (pystac.Catalog, pystac.Collection)):
+            raise KeyError(
+                "'recursive' is only a valid option for "
+                "pystac.Catalogs and pystac.Collections"
+            )
+        stac_object.fully_resolve()
+        if show_diff:
+            for _, children, items in stac_object.walk():
+                for obj in chain(children, items):
+                    _print_diff(obj)
+        if save:
+            stac_object.save()
+    else:
+        if save:
+            stac_object.save_object()
+
+    return stac_object

tests/cli/commands/test_migrate.py

@@ -0,0 +1,164 @@
+import shutil
+from pathlib import Path
+
+import pystac
+import pytest
+from click.testing import CliRunner
+from stactools.cli.cli import cli
+
+from tests import test_data
+
+
+@pytest.fixture(scope="function")
+def tmp_planet_disaster_path(tmp_path: Path) -> str:
+    src = test_data.get_path("data-files/planet-disaster-v1.0.0-beta.2")
+    dst = tmp_path / "planet-disaster"
+    shutil.copytree(src, str(dst))
+    return str(dst / "collection.json")
+
+
+def test_migrate_no_save_by_default(tmp_planet_disaster_path: str):
+    path = tmp_planet_disaster_path
+    with open(path) as f:
+        before = f.readlines()
+
+    runner = CliRunner()
+    result = runner.invoke(cli, ["migrate", path])
+    assert result.exit_code == 0
+
+    with open(path) as f:
+        after = f.readlines()
+
+    assert before == after
+
+
+def test_migrate_with_save_no_recursive(tmp_planet_disaster_path: str):
+    path = tmp_planet_disaster_path
+    root = pystac.Collection.from_file(path)
+    child_path = next(root.get_children()).get_self_href()
+    item_path = next(root.get_all_items()).get_self_href()
+
+    with open(path) as f:
+        root_before = f.readlines()
+
+    with open(child_path) as f:
+        child_before = f.readlines()
+
+    with open(item_path) as f:
+        item_before = f.readlines()
+
+    runner = CliRunner()
+    result = runner.invoke(cli, ["migrate", path, "--save"])
+    assert result.exit_code == 0
+
+    with open(path) as f:
+        root_after = f.readlines()
+
+    with open(child_path) as f:
+        child_after = f.readlines()
+
+    with open(item_path) as f:
+        item_after = f.readlines()
+
+    assert root_before != root_after, path
+    assert child_before == child_after, child_path
+    assert item_before == item_after, item_path
+
+
+def test_migrate_with_save_and_recursive(tmp_planet_disaster_path: str):
+    path = tmp_planet_disaster_path
+    root = pystac.Collection.from_file(path)
+    child_path = next(root.get_children()).get_self_href()
+    item_path = next(root.get_all_items()).get_self_href()
+
+    with open(path) as f:
+        root_before = f.readlines()
+
+    with open(child_path) as f:
+        child_before = f.readlines()
+
+    with open(item_path) as f:
+        item_before = f.readlines()
+
+    runner = CliRunner()
+    result = runner.invoke(cli, ["migrate", path, "-r", "-s"])
+    assert result.exit_code == 0
+
+    with open(path) as f:
+        root_after = f.readlines()
+
+    with open(child_path) as f:
+        child_after = f.readlines()
+
+    with open(item_path) as f:
+        item_after = f.readlines()
+
+    assert root_before != root_after, path
+    assert child_before != child_after, child_path
+    assert item_before != item_after, item_path
+
+
+def test_migrate_show_diff(tmp_planet_disaster_path: str):
+    path = tmp_planet_disaster_path
+    root = pystac.Collection.from_file(path)
+    child_path = next(root.get_children()).get_self_href()
+    item_path = next(root.get_all_items()).get_self_href()
+
+    runner = CliRunner()
+    result = runner.invoke(cli, ["migrate", path, "--show-diff"])
+    assert result.exit_code == 0
+
+    assert result.output.startswith("--- a")
+    assert path in result.output
+    assert child_path not in result.output
+    assert item_path not in result.output
+
+
+def test_migrate_show_diff_and_recursive(tmp_planet_disaster_path: str):
+    path = tmp_planet_disaster_path
+    root = pystac.Collection.from_file(path)
+    child_path = next(root.get_children()).get_self_href()
+    item_path = next(root.get_all_items()).get_self_href()
+
+    runner = CliRunner()
+    result = runner.invoke(cli, ["migrate", path, "--show-diff", "--recursive"])
+    assert result.exit_code == 0
+
+    assert result.output.startswith("--- a")
+    assert path in result.output
+    assert child_path in result.output
+    assert item_path in result.output
+
+
+def test_migrate_hide_diff_with_no_save_raises(tmp_planet_disaster_path: str):
+    path = tmp_planet_disaster_path
+
+    runner = CliRunner()
+    result = runner.invoke(cli, ["migrate", path, "--hide-diff"])
+    assert result.exit_code == 2
+    assert (
+        "Error: It is only valid to use 'hide-diff' when 'save' is enabled "
+        "otherwise there would be no output." in result.output
+    )
+
+
+def test_migrate_hide_diff(tmp_planet_disaster_path: str):
+    path = tmp_planet_disaster_path
+
+    runner = CliRunner()
+    result = runner.invoke(cli, ["migrate", path, "--hide-diff", "--save"])
+    assert not result.output
+
+
+def test_migrate_recursive_invalid_for_items(tmp_planet_disaster_path: str):
+    path = tmp_planet_disaster_path
+    root = pystac.Collection.from_file(path)
+    item_path = next(root.get_all_items()).get_self_href()
+
+    runner = CliRunner()
+    result = runner.invoke(cli, ["migrate", item_path, "-r"])
+    assert result.exit_code == 2
+    assert (
+        "Error: 'recursive' is only a valid option for "
+        "pystac.Catalogs and pystac.Collections" in result.output
+    )