CLI: native collections support (#1197)

* cli: auto-append collections adaptor

* changeset

* track collections on credentials

* cli: execute with collections

* update command aliases

* make api-token/pat usage more consistent

* lightning mock: add  very basic collections support

* changeset

* Collections integration test (#1199)

* attempt test

* fix test

* tidying

* reduce minimum release age and install

* fix lockfile

* remove adaptor override

* package lock again

* more cleanup

* remove another wierd local dep

* package lock yet again

* fix an option conflict

* fix the mock

* update mock to better match lightning

Author: josephjclark

.changeset/brave-islands-search.md

@@ -0,0 +1,5 @@
+---
+'@openfn/cli': minor
+---
+
+Support aliases on all project subcommands

.changeset/tiny-forks-brake.md

@@ -0,0 +1,5 @@
+---
+'@openfn/cli': minor
+---
+
+Auto-load collections adptor when using collections

.changeset/wicked-plants-push.md

@@ -0,0 +1,5 @@
+---
+'@openfn/lightning-mock': minor
+---
+
+Add basic collections support (GET only)

integration-tests/cli/test/execute-workflow.test.ts

@@ -1,9 +1,24 @@
 import test from 'ava';
 import { rm, mkdir } from 'node:fs/promises';
 import path from 'node:path';
+
+import createLightningServer from '@openfn/lightning-mock';
+
 import run from '../src/run';
 import { getJSON } from '../src/util';
 
+// set up a lightning mock
+let server: any;
+
+const port = 8968;
+
+test.before(async () => {
+  server = await createLightningServer({ port });
+  server.collections.createCollection('stuff');
+  // Important: the collection value MUST be as string
+  server.collections.upsert('stuff', 'x', JSON.stringify({ id: 'x' }));
+});
+
 const jobsPath = path.resolve('test/fixtures');
 
 // Note that these tests are STATEFUL
@@ -151,7 +166,6 @@ test.serial(
   `openfn ${jobsPath}/wf-creds.json --credentials ${jobsPath}/creds.json`,
   async (t) => {
     const { err, stdout, stderr } = await run(t.title);
-    console.log({ stdout, stderr });
     t.falsy(err);
 
     const out = getJSON();
@@ -285,3 +299,16 @@ test.serial(
     });
   }
 );
+
+// collections basic test
+test.serial(
+  `openfn ${jobsPath}/collections.json --endpoint http://localhost:${port} --api-key xyz`,
+  async (t) => {
+    const { err } = await run(t.title);
+    t.falsy(err);
+
+    const out = getJSON();
+
+    t.deepEqual(out.data, { id: 'x' });
+  }
+);

integration-tests/cli/test/fixtures/collections.json

@@ -0,0 +1,10 @@
+{
+  "workflow": {
+    "steps": [
+      {
+        "adaptor": "common",
+        "expression": "collections.get('stuff', 'x')"
+      }
+    ]
+  }
+}

packages/cli/src/collections/command.ts

@@ -88,14 +88,6 @@ const key = {
   },
 };
 
-const token = {
-  name: 'pat',
-  yargs: {
-    alias: ['token'],
-    description: 'Lightning Personal Access Token (PAT)',
-  },
-};
-
 const endpoint = {
   name: 'endpoint',
   yargs: {
@@ -160,7 +152,7 @@ const updatedAfter = {
 const getOptions = [
   collectionName,
   key,
-  token,
+  o.apiKey,
   endpoint,
   pageSize,
   limit,
@@ -201,7 +193,7 @@ const dryRun = {
 const removeOptions = [
   collectionName,
   key,
-  token,
+  o.apiKey,
   endpoint,
   dryRun,
 
@@ -243,7 +235,7 @@ const setOptions = [
   override(key as any, {
     demand: false,
   }),
-  token,
+  o.apiKey,
   endpoint,
   value,
   items,

packages/cli/src/execute/command.ts

@@ -1,19 +1,22 @@
 import yargs from 'yargs';
-import { build, ensure } from '../util/command-builders';
+import { build, ensure, override } from '../util/command-builders';
 import * as o from '../options';
 
 import type { Opts } from '../options';
 
 export type ExecuteOptions = Required<
   Pick<
     Opts,
+    | 'apiKey'
     | 'adaptors'
     | 'autoinstall'
     | 'baseDir'
     | 'cacheSteps'
     | 'command'
     | 'compile'
     | 'credentials'
+    | 'collectionsEndpoint'
+    | 'collectionsVersion'
     | 'expandAdaptors'
     | 'end'
     | 'immutable'
@@ -44,10 +47,16 @@ const options = [
   o.expandAdaptors, // order is important
 
   o.adaptors,
+  override(o.apiKey, {
+    description: 'API token for collections',
+    alias: ['collections-api-key', 'collections-token', 'pat'],
+  }),
   o.autoinstall,
   o.cacheSteps,
   o.compile,
   o.credentials,
+  o.collectionsEndpoint,
+  o.collectionsVersion,
   o.end,
   o.ignoreImports,
   o.immutable,

packages/cli/src/options.ts

@@ -31,6 +31,8 @@ export type Opts = {
   configPath?: string;
   confirm?: boolean;
   credentials?: string;
+  collectionsEndpoint?: string;
+  collectionsVersion?: string;
   describe?: string;
   end?: string; // workflow end node
   expandAdaptors?: boolean; // for unit tests really
@@ -136,12 +138,17 @@ export const autoinstall: CLIOption = {
   },
 };
 
-export const apikey: CLIOption = {
+export const apiKey: CLIOption = {
   name: 'apikey',
   yargs: {
-    alias: ['key', 'pat', 'token'],
+    alias: ['pat', 'token', 'api-key'],
     description:
-      '[beta only] API Key, Personal Access Token (Pat), or other access token',
+      'API Key, Personal Access Token (PAT), or other access token from Lightning',
+  },
+  ensure: (opts: any) => {
+    if (!opts.apikey) {
+      opts.apiKey = process.env.OPENFN_API_KEY;
+    }
   },
 };
 
@@ -240,6 +247,23 @@ export const configPath: CLIOption = {
   },
 };
 
+export const collectionsVersion: CLIOption = {
+  name: 'collections-version',
+  yargs: {
+    description:
+      'The version of the collections adaptor to use. Defaults to latest. Use OPENFN_COLLECTIONS_VERSION env.',
+  },
+};
+
+export const collectionsEndpoint: CLIOption = {
+  name: 'collections-endpoint',
+  yargs: {
+    alias: ['endpoint'],
+    description:
+      'The Lightning server to use for collections. Will use the project endpoint if available. Use OPENFN_COLLECTIONS_ENDPOINT env.',
+  },
+};
+
 export const credentials: CLIOption = {
   name: 'credentials',
   yargs: {

packages/cli/src/projects/fetch.ts

@@ -35,7 +35,7 @@ export type FetchOptions = Pick<
 
 const options = [
   po.alias,
-  o.apikey,
+  o.apiKey,
   o.endpoint,
   o.log,
   o.logJson,

packages/cli/src/projects/pull.ts

@@ -30,7 +30,7 @@ const options = [
   o2.workspace,
 
   // general options
-  o.apikey,
+  o.apiKey,
   o.endpoint,
   o.log,
   override(o.path, {