Add -t/--traverse option to fedify lookup

Close #195

Author: dahlia

CHANGES.md

@@ -8,6 +8,17 @@ Version 1.4.0
 
 To be released.
 
+ -  The `suppressError` option of Activity Vocabulary APIs,
+    `traverseCollection()` function, and `Context.traverseCollection()` method
+    now suppresses errors occurred JSON-LD processing.
+
+ -  Added `-t`/`--traverse` option to the `fedify lookup` subcommand.  [[#195]]
+
+ -  Added `-S`/`--suppress-errors` option to the `fedify lookup` subcommand.
+    [[#195]]
+
+[#195]: https://github.com/dahlia/fedify/issues/195
+
 
 Version 1.3.1
 -------------

cli/lookup.ts

@@ -2,28 +2,44 @@ import { colors } from "@cliffy/ansi";
 import { Command } from "@cliffy/command";
 import {
   Application,
+  Collection,
   CryptographicKey,
   type DocumentLoader,
   generateCryptoKeyPair,
   getAuthenticatedDocumentLoader,
+  type Link,
   lookupObject,
   type Object,
   type ResourceDescriptor,
   respondWithObject,
+  traverseCollection,
 } from "@fedify/fedify";
+import { getLogger } from "@logtape/logtape";
 import ora from "ora";
 import { getContextLoader, getDocumentLoader } from "./docloader.ts";
 import { spawnTemporaryServer, type TemporaryServer } from "./tempserver.ts";
 import { printJson } from "./utils.ts";
 
+const logger = getLogger(["fedify", "cli", "lookup"]);
+
 export const command = new Command()
   .arguments("<...urls:string>")
   .description(
     "Lookup an Activity Streams object by URL or the actor handle.  " +
       "The argument can be either a URL or an actor handle " +
-      "(e.g., @username@domain).",
+      "(e.g., @username@domain), and it can be multiple.",
   )
   .option("-a, --authorized-fetch", "Sign the request with an one-time key.")
+  .option(
+    "-t, --traverse",
+    "Traverse the given collection to fetch all items.  If it is turned on, " +
+      "the argument cannot be multiple.",
+  )
+  .option(
+    "-S, --suppress-errors",
+    "Suppress partial errors while traversing the collection.",
+    { depends: ["traverse"] },
+  )
   .option("-r, --raw", "Print the fetched JSON-LD document as is.", {
     conflicts: ["compact", "expand"],
   })
@@ -36,12 +52,24 @@ export const command = new Command()
   .option("-u, --user-agent <string>", "The custom User-Agent header value.")
   .option(
     "-s, --separator <string>",
-    "Specify the separator between adjacent output objects.",
+    "Specify the separator between adjacent output objects or " +
+      "collection items.",
     { default: "----" },
   )
   .action(async (options, ...urls: string[]) => {
+    if (urls.length < 1) {
+      console.error("At least one URL or actor handle must be provided.");
+      Deno.exit(1);
+    } else if (options.traverse && urls.length > 1) {
+      console.error(
+        "The -t/--traverse option cannot be used with multiple arguments.",
+      );
+      Deno.exit(1);
+    }
     const spinner = ora({
-      text: "Looking up the object...",
+      text: `Looking up the ${
+        options.traverse ? "collection" : urls.length > 1 ? "objects" : "object"
+      }...`,
       discardStdin: false,
     }).start();
     let server: TemporaryServer | undefined = undefined;
@@ -95,9 +123,84 @@ export const command = new Command()
         privateKey: key.privateKey,
       });
     }
-    spinner.text = urls.length > 1
-      ? "Looking up objects..."
-      : "Looking up an object...";
+    spinner.text = `Looking up the ${
+      options.traverse ? "collection" : urls.length > 1 ? "objects" : "object"
+    }...`;
+
+    async function printObject(object: Object | Link): Promise<void> {
+      if (options.raw) {
+        printJson(await object.toJsonLd({ contextLoader }));
+      } else if (options.compact) {
+        printJson(
+          await object.toJsonLd({ format: "compact", contextLoader }),
+        );
+      } else if (options.expand) {
+        printJson(
+          await object.toJsonLd({ format: "expand", contextLoader }),
+        );
+      } else {
+        console.log(object);
+      }
+    }
+
+    if (options.traverse) {
+      const url = urls[0];
+      const collection = await lookupObject(url, {
+        documentLoader: authLoader ?? documentLoader,
+        contextLoader,
+        userAgent: options.userAgent,
+      });
+      if (collection == null) {
+        spinner.fail(`Failed to fetch object: ${colors.red(url)}.`);
+        if (authLoader == null) {
+          console.error(
+            "It may be a private object.  Try with -a/--authorized-fetch.",
+          );
+        }
+        await server?.close();
+        Deno.exit(1);
+      }
+      if (!(collection instanceof Collection)) {
+        spinner.fail(
+          `Not a collection: ${colors.red(url)}.  ` +
+            "The -t/--traverse option requires a collection.",
+        );
+        await server?.close();
+        Deno.exit(1);
+      }
+      spinner.succeed(`Fetched collection: ${colors.green(url)}.`);
+      try {
+        let i = 0;
+        for await (
+          const item of traverseCollection(collection, {
+            documentLoader: authLoader ?? documentLoader,
+            contextLoader,
+            suppressError: options.suppressErrors,
+          })
+        ) {
+          if (i > 0) console.log(options.separator);
+          printObject(item);
+          i++;
+        }
+      } catch (error) {
+        logger.error("Failed to complete the traversal: {error}", { error });
+        spinner.fail("Failed to complete the traversal.");
+        if (authLoader == null) {
+          console.error(
+            "It may be a private object.  Try with -a/--authorized-fetch.",
+          );
+        } else {
+          console.error(
+            "Use the -S/--suppress-errors option to suppress partial errors.",
+          );
+        }
+        await server?.close();
+        Deno.exit(1);
+      }
+      spinner.succeed("Successfully fetched all items in the collection.");
+      await server?.close();
+      Deno.exit(0);
+    }
 
     const promises: Promise<Object | null>[] = [];
     for (const url of urls) {
@@ -131,19 +234,7 @@ export const command = new Command()
           success = false;
         } else {
           spinner.succeed(`Fetched object: ${colors.green(url)}.`);
-          if (options.raw) {
-            printJson(await object.toJsonLd({ contextLoader }));
-          } else if (options.compact) {
-            printJson(
-              await object.toJsonLd({ format: "compact", contextLoader }),
-            );
-          } else if (options.expand) {
-            printJson(
-              await object.toJsonLd({ format: "expand", contextLoader }),
-            );
-          } else {
-            console.log(object);
-          }
+          printObject(object);
           if (i < urls.length - 1) {
             console.log(options.separator);
           }

docs/cli.md

@@ -305,6 +305,47 @@ As you can see, the outputs are separated by `----` by default.  You can change
 the separator by using the [`-s`/`--separator`](#s-separator-output-separator)
 option.
 
+> [!NOTE]
+> The `fedify lookup` command cannot take multiple argument if
+> [`-t`/`--traverse`](#t-traverse-traverse-the-collection) option is turned
+> on.
+
+### `-t`/`--traverse`: Traverse the collection
+
+*This option is available since Fedify 0.14.0.*
+
+The `-t`/`--traverse` option is used to traverse the collection when looking up
+a collection object.  For example, the below command looks up a collection
+object:
+
+~~~~ sh
+fedify lookup --traverse https://fosstodon.org/users/hongminhee/outbox
+~~~~
+
+The difference between with and without the `-t`/`--traverse` option is that
+the former will output the objects in the collection, while the latter will
+output the collection object itself.
+
+This option only works with a single argument, and it has to be a collection.
+
+### `-S`/`--suppress-errors`: Suppress partial errors during traversal
+
+*This option is available since Fedify 0.14.0.*
+
+The `-S`/`--suppress-errors` option is used to suppress partial errors during
+traversal.  For example, the below command looks up a collection object with
+the `-t`/`--traverse` option:
+
+~~~~ sh
+fedify lookup --traverse --suppress-errors https://fosstodon.org/users/hongminhee/outbox
+~~~~
+
+The difference between with and without the `-S`/`--suppress-errors` option is
+that the former will suppress the partial errors during traversal, while the
+latter will stop the traversal when an error occurs.
+
+This option depends on the `-t`/`--traverse` option.
+
 ### `-c`/`--compact`: Compact JSON-LD
 
 > [!NOTE]
@@ -692,6 +733,10 @@ fedify lookup -s ==== @[email protected] @[email protected]
 
 It does not affect the output when looking up a single object.
 
+> [!TIP]
+> The separator is also used when looking up a collection object with the
+> [`-t`/`--traverse`](#t-traverse-traverse-the-collection) option.
+
 
 `fedify inbox`: Ephemeral inbox server
 --------------------------------------