Merge pull request #1146 from OpenFn/release/next

Release/next

Author: josephjclark

.github/workflows/project-integration-tests.yaml

@@ -3,9 +3,9 @@ name: Project Integration
 # This workflow runs on every release branch,
 # when a PR is labeled with 'run_project_tests',
 on:
-  push:
-    branches:
-      - 'release/**'
+  # push:
+  #   branches:
+  #     - 'release/**'
   pull_request:
     types: [labeled]
 

.gitignore

@@ -1,3 +1,5 @@
+.env
+
 # dev
 *.code-workspace
 examples/**/dist
@@ -35,3 +37,5 @@ tmp
 
 *.drawio.bkp
 .fuse
+
+

…tegration-tests/cli/test/project.test.ts …ration-tests/cli/test/project-v1.test.tsintegration-tests/cli/test/project.test.ts renamed to integration-tests/cli/test/project-v1.test.ts integration-tests/cli/test/project.test.ts renamed to integration-tests/cli/test/project-v1.test.ts

@@ -3,6 +3,8 @@ import { rm, mkdir, writeFile, readFile } from 'node:fs/promises';
 import path from 'node:path';
 import run from '../src/run';
 
+// These tests use the legacy v1 yaml structure
+
 const mainYaml = `
 id: 8dbc4349-52b4-4bf2-be10-fdf06da52c46
 name: hello-world
@@ -107,6 +109,15 @@ test.before(async () => {
   );
 });
 
+test.serial('list available projects', async (t) => {
+  const { stdout } = await run(`openfn projects -p ${projectsPath}`);
+
+  t.regex(stdout, /hello-world/);
+  t.regex(stdout, /8dbc4349-52b4-4bf2-be10-fdf06da52c46/);
+  t.regex(stdout, /hello-world-staging/);
+  t.regex(stdout, /5deddbfa-c63f-4dbc-98b5-a49d3395a488/);
+});
+
 // checkout a project from a yaml file
 test.serial('Checkout a project', async (t) => {
   await run(`openfn checkout hello-world -p ${projectsPath}`);
@@ -128,8 +139,6 @@ steps:
       transform-data:
         disabled: false
         condition: true
-        openfn:
-          uuid: 33dce70f-047f-4508-82fd-950eb508519b
   - id: transform-data
     name: Transform data
     adaptor: "@openfn/language-common@latest"

integration-tests/cli/test/project-v2.test.ts

@@ -0,0 +1,197 @@
+import test from 'ava';
+import { rm, mkdir, writeFile, readFile } from 'node:fs/promises';
+import path from 'node:path';
+import run from '../src/run';
+
+const mainYaml = `
+id: sandboxing-simple
+name: Sandboxing Simple
+version: 2
+collections: []
+credentials:
+  - id: 10a50683-78b0-4ddf-9c14-23a1fb21074a
+    name: name
+    owner: editor@openfn.org
+openfn:
+  uuid: a272a529-716a-4de7-a01c-a082916c6d23
+  endpoint: http://localhost:4000
+  env: project
+  fetched_at: 2025-11-26T17:55:09.716Z
+  inserted_at: 2025-10-17T10:30:44Z
+  updated_at: 2025-10-24T14:52:13Z
+options:
+  env: main
+  allow_support_access: false
+  requires_mfa: false
+  retention_policy: retain_all
+  version_history: []
+workflows:
+  - name: Hello Workflow
+    steps:
+      - id: trigger
+        type: webhook
+        openfn:
+          enabled: true
+          uuid: 9c0a4e8a-b82f-4fa5-8d12-419da143cd04
+        next:
+          transform-data:
+            disabled: false
+            condition: true
+            openfn:
+              uuid: add150e9-8616-48ca-844e-8aaa489c7a10
+      - id: transform-data
+        name: Transform data
+        expression: |-
+          // TODO
+        adaptor: "@openfn/language-dhis2@8.0.4"
+        openfn:
+          uuid: a9f64216-7974-469d-8415-d6d9baf2f92e
+          project_credential_id: null
+    openfn:
+      uuid: af697653-98d2-46e4-912f-e1cb6bf8f4b4
+      concurrency: 5
+      inserted_at: 2025-10-17T10:30:51Z
+      updated_at: 2025-10-24T14:52:13Z
+      deleted_at: null
+      lock_version: 16
+    id: hello-workflow
+    history: []
+`;
+
+const stagingYaml = `id: staging
+name: staging
+version: 2
+collections: []
+credentials:
+  - id: 07c0baa7-c1d7-44b2-abb6-3446defbe3e3
+    name: name
+    owner: editor@openfn.org
+openfn:
+  uuid: bc6629fb-7dc8-4b28-93af-901e2bd58dc4
+  endpoint: http://localhost:4000
+  env: staging
+  fetched_at: 2025-11-27T11:40:37.670Z
+  inserted_at: 2025-11-27T11:39:33Z
+  updated_at: 2025-11-27T11:39:33Z
+options:
+  env: main
+  color: "#F39B33"
+  parent_id: a272a529-716a-4de7-a01c-a082916c6d23
+  allow_support_access: false
+  requires_mfa: false
+  retention_policy: retain_all
+  version_history:
+    - 7b0f5af558f5
+workflows:
+  - name: Hello Workflow
+    steps:
+      - id: trigger
+        type: webhook
+        openfn:
+          enabled: false
+          uuid: 9ef55dca-16a3-480c-a807-2d37744e6e53
+        next:
+          transform-data:
+            disabled: false
+            condition: true
+            openfn:
+              uuid: f34146b5-de43-4b05-ac00-3b4f327e62ec
+      - id: transform-data
+        name: Transform data
+        expression: |-
+          fn()
+        adaptor: "@openfn/language-dhis2@8.0.4"
+        openfn:
+          uuid: 5b4c74f9-76ac-4715-bd45-04b130ca549c
+          project_credential_id: null
+     
+    openfn:
+      uuid: 10ce2914-16aa-4e00-b746-47678b1c60d4
+      concurrency: 5
+      inserted_at: 2025-11-27T11:39:33Z
+      updated_at: 2025-11-27T11:39:47Z
+      deleted_at: null
+      lock_version: 1
+    id: hello-workflow
+    history: []
+`;
+const projectsPath = path.resolve('tmp/project');
+
+test.before(async () => {
+  await rm('tmp/project', { recursive: true });
+  await mkdir('tmp/project/.projects', { recursive: true });
+
+  await writeFile('tmp/project/openfn.yaml', '');
+  await writeFile('tmp/project/.projects/main@app.openfn.org.yaml', mainYaml);
+  await writeFile(
+    'tmp/project/.projects/staging@app.openfn.org.yaml',
+    stagingYaml
+  );
+});
+
+test.serial('list available projects', async (t) => {
+  const { stdout } = await run(`openfn projects -p ${projectsPath}`);
+  t.regex(stdout, /sandboxing-simple/);
+  t.regex(stdout, /a272a529-716a-4de7-a01c-a082916c6d23/);
+  t.regex(stdout, /staging/);
+  t.regex(stdout, /bc6629fb-7dc8-4b28-93af-901e2bd58dc4/);
+});
+
+test.serial('Checkout a project', async (t) => {
+  await run(`openfn checkout staging -p ${projectsPath}`);
+
+  // check workflow.yaml
+  const workflowYaml = await readFile(
+    path.resolve(projectsPath, 'workflows/hello-workflow/hello-workflow.yaml'),
+    'utf8'
+  );
+  t.is(
+    workflowYaml,
+    `id: hello-workflow
+name: Hello Workflow
+options:
+  history: []
+steps:
+  - id: trigger
+    type: webhook
+    next:
+      transform-data:
+        disabled: false
+        condition: true
+  - id: transform-data
+    name: Transform data
+    adaptor: "@openfn/language-dhis2@8.0.4"
+    expression: ./transform-data.js
+`
+  );
+
+  const expr = await readFile(
+    path.resolve(projectsPath, 'workflows/hello-workflow/transform-data.js'),
+    'utf8'
+  );
+  t.is(expr.trim(), 'fn()');
+});
+
+// requires the prior test to run
+test.serial('merge a project', async (t) => {
+  const readStep = () =>
+    readFile(
+      path.resolve(projectsPath, 'workflows/hello-workflow/transform-data.js'),
+      'utf8'
+    ).then((str) => str.trim());
+
+  await run(`openfn checkout sandboxing-simple -p ${projectsPath}`);
+
+  // assert the initial step code
+  const initial = await readStep();
+  t.is(initial, '// TODO');
+
+  // Run the merge
+  const { stdout } = await run(
+    `openfn merge staging -p ${projectsPath} --force`
+  );
+
+  // Check the step is updated
+  const merged = await readStep();
+  t.is(merged, 'fn()');
+});

packages/cli/CHANGELOG.md

@@ -1,5 +1,23 @@
 # @openfn/cli
 
+## 1.19.0
+
+### Minor Changes
+
+- 72f18b1: In pull --beta, use an improved and updated structure for project.yaml files
+- 9001244: Support .env files in all CLI commands
+
+### Patch Changes
+
+- Updated dependencies [72f18b1]
+- Updated dependencies [2cc28a6]
+  - @openfn/lexicon@1.2.7
+  - @openfn/project@0.9.1
+  - @openfn/logger@1.1.0
+  - @openfn/compiler@1.2.1
+  - @openfn/deploy@0.11.4
+  - @openfn/runtime@1.7.6
+
 ## 1.18.6
 
 ### Patch Changes

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.18.6",
+  "version": "1.19.0",
   "description": "CLI devtools for the OpenFn toolchain",
   "engines": {
     "node": ">=18",
@@ -57,6 +57,8 @@
     "@openfn/project": "workspace:^",
     "@openfn/runtime": "workspace:*",
     "chalk": "^5.6.2",
+    "dotenv": "^17.2.3",
+    "dotenv-expand": "^12.0.3",
     "figures": "^5.0.0",
     "rimraf": "^6.0.1",
     "treeify": "^1.1.0",

packages/cli/src/checkout/handler.ts

@@ -40,7 +40,7 @@ const checkoutHandler = async (options: CheckoutOptions, logger: Logger) => {
   await rimraf(path.join(commandPath, config.workflowRoot ?? 'workflows'));
 
   // expand project into directory
-  const files = switchProject.serialize('fs');
+  const files: any = switchProject.serialize('fs');
   for (const f in files) {
     if (files[f]) {
       fs.mkdirSync(path.join(commandPath, path.dirname(f)), {

packages/cli/src/cli.ts

@@ -1,6 +1,7 @@
 import yargs, { Arguments } from 'yargs';
 import { hideBin } from 'yargs/helpers';
 
+import loadDotEnv from './env';
 import apolloCommand from './apollo/command';
 import collectionsCommand from './collections/command';
 import compileCommand from './compile/command';
@@ -18,6 +19,13 @@ import checkoutCommand from './checkout/command';
 import mergeCommand from './merge/command';
 import workflowVersionCommand from './version/command';
 
+const env = loadDotEnv();
+if (env) {
+  // Write the env so that the inner process can use it
+  // This is just a convenience for logging back to the user
+  process.env.$DOT_ENV_OVERRIDES = Object.keys(env).join(',');
+}
+
 const y = yargs(hideBin(process.argv));
 
 export const cmd = y

packages/cli/src/commands.ts

@@ -21,6 +21,7 @@ import mapAdaptorsToMonorepo, {
 } from './util/map-adaptors-to-monorepo';
 import printVersions from './util/print-versions';
 import abort from './util/abort';
+import { report } from './env';
 
 export type CommandList =
   | 'apollo'
@@ -80,6 +81,9 @@ const parse = async (options: Opts, log?: Logger) => {
     await printVersions(logger, options);
   }
 
+  // Tell the user whether we're using env vars
+  report(logger);
+
   const { monorepoPath } = options;
   if (monorepoPath) {
     // TODO how does this occur?

packages/cli/src/deploy/beta.ts

@@ -29,6 +29,8 @@ export async function handler(options: DeployOptionsBeta, logger: Logger) {
   // TMP use options.path to set the directory for now
   // We'll need to manage this a bit better
   const project = await Project.from('fs', { root: options.path || '.' });
+  // Why is there an id on openfn here?
+  console.log({ openfn: project.openfn });
 
   // TODO: work out if there's any diff