Skip to content

Commit 4cc799b

Browse files
josephjclarkjosephjclark
authored
Add Project aliases with fuzzy matching (#1167)
* refactor to allow alias * updates to alias * add fuzzy alias function with tests * more tests * handle id conflict * add error * Workspace integration * increase test timeout * increase timeout again * increase timeout specifically on command tests * rework alias to only be in the file name (and rewrite fuzzy matcher) * simplify * remove ai comments * started refactoring but getting lost - its over complex. deferring until lateR * finish refactor. phew * get pull working * tweak active project and list cli * fix cli fetch to allow aliases * Better error handling for fetch * checkout tests * fix tests * some test fixes * typos * changeset & tidyup * fixing * total refactor of fetch around aliasing gulp * update test * fetch: more tests (and fixes) * type hack better fix incoming on adifferent branch * first atteempt at unit tests This uses undici mocks but ofcourse the CLI runs out of a different proces, so the mocked endpoints don't return I'll need to refactor to use the lightning mock * accept and save real data in the provisioner API * dev api * add one passing integration test * one more test for the road * more tests * checkout test * update checkout * update tests * export default project id for testing * fix pull test Changes to the lightning mock caused it to break * comments * fix integration test * fix deploy test again * one last test fix * major refactor of new fetch and new test souite next: consolidate test files, closely revew, and manual test * consolidate tsts * tweak log output
1 parent 3e63c08 commit 4cc799b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+2081
-662
lines changed

‎.changeset/breezy-walls-bet.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@openfn/lightning-mock': minor
3+
---
4+
5+
Update the provisioner API to support real data

‎.changeset/four-dots-repeat.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@openfn/project': minor
3+
---
4+
5+
Add support for aliases (replaces env)

‎.changeset/many-baboons-love.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@openfn/cli': patch
3+
---
4+
5+
Refactor pull into a project command

‎.changeset/some-tires-create.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@openfn/project': patch
3+
---
4+
5+
Project: remove `getIdentifier()` in favour of `qname` (qualified name)

‎integration-tests/cli/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
},
1717
"dependencies": {
1818
"@openfn/lightning-mock": "workspace:^",
19+
"@openfn/project": "workspace:*",
1920
"@types/node": "^18.19.127",
2021
"ava": "5.3.1",
2122
"date-fns": "^2.30.0",

‎integration-tests/cli/test/deploy.test.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import test from 'ava';
22
import run from '../src/run';
3-
import createLightningServer from '@openfn/lightning-mock';
3+
import createLightningServer, {
4+
DEFAULT_PROJECT_ID,
5+
} from '@openfn/lightning-mock';
46
import { extractLogs, assertLog } from '../src/util';
57
import { rimraf } from 'rimraf';
68

@@ -18,7 +20,7 @@ test.before(async () => {
1820

1921
// This should fail against the built CLI right now
2022
test.serial(
21-
`OPENFN_ENDPOINT=${endpoint} openfn pull 123 --log-json`,
23+
`OPENFN_ENDPOINT=${endpoint} openfn pull ${DEFAULT_PROJECT_ID} --log-json`,
2224
async (t) => {
2325
const { stdout, stderr } = await run(t.title);
2426
t.falsy(stderr);

‎integration-tests/cli/test/sync.test.ts‎

Lines changed: 287 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,287 @@
1+
import test from 'ava';
2+
import path from 'node:path';
3+
import fs from 'node:fs';
4+
import run from '../src/run';
5+
import { generateProject } from '@openfn/project';
6+
import createLightningServer from '@openfn/lightning-mock';
7+
import { rimraf } from 'rimraf';
8+
9+
let PORT = 5353;
10+
let lightning;
11+
const endpoint = `http://localhost:${PORT}/api/provision`;
12+
13+
test.before(async () => {
14+
await rimraf('tmp/sync');
15+
16+
lightning = createLightningServer({
17+
port: PORT,
18+
});
19+
});
20+
21+
const initWorkspace = (t: any) => {
22+
const id = t.title.replaceAll(' ', '_').toLowerCase();
23+
const p = path.resolve('tmp/sync', id);
24+
25+
return {
26+
workspace: p,
27+
read: (filePath: string) => {
28+
return fs.readFileSync(path.resolve(p, filePath), 'utf8');
29+
},
30+
};
31+
};
32+
33+
const gen = (name = 'patients', workflows = ['trigger-job(body="fn()")']) => {
34+
// generate a project
35+
const project = generateProject(name, workflows, {
36+
openfnUuid: true,
37+
});
38+
const state = project.serialize('state', { format: 'json' });
39+
lightning.addProject(state);
40+
return project;
41+
};
42+
43+
test('fetch a new project', async (t) => {
44+
const { workspace, read } = initWorkspace(t);
45+
const project = gen();
46+
47+
await run(
48+
`openfn project fetch \
49+
--workspace ${workspace} \
50+
--endpoint ${endpoint} \
51+
--api-key abc \
52+
${project.openfn.uuid}`
53+
);
54+
55+
// now check that the filesystem is roughly right
56+
const pyaml = read('.projects/main@localhost.yaml');
57+
58+
t.regex(pyaml, /id: patients/);
59+
t.regex(pyaml, new RegExp(`uuid: ${project.openfn.uuid}`));
60+
});
61+
62+
test('fetch a new project with an alias', async (t) => {
63+
const { workspace, read } = initWorkspace(t);
64+
const project = gen();
65+
66+
await run(
67+
`openfn project fetch \
68+
--workspace ${workspace} \
69+
--endpoint ${endpoint} \
70+
--api-key abc \
71+
--alias staging\
72+
${project.openfn.uuid}`
73+
);
74+
75+
// now check that the filesystem is roughly right
76+
const pyaml = read('.projects/staging@localhost.yaml');
77+
78+
t.regex(pyaml, /id: patients/);
79+
t.regex(pyaml, new RegExp(`uuid: ${project.openfn.uuid}`));
80+
});
81+
82+
test('fetch a new project to a path', async (t) => {
83+
const { workspace, read } = initWorkspace(t);
84+
const project = gen();
85+
86+
await run(
87+
`openfn project fetch \
88+
--workspace ${workspace} \
89+
--endpoint ${endpoint} \
90+
--api-key abc \
91+
--output ${workspace}/project.yaml\
92+
${project.openfn.uuid}`
93+
);
94+
95+
// now check that the filesystem is roughly right
96+
const pyaml = read('project.yaml');
97+
98+
t.regex(pyaml, /id: patients/);
99+
t.regex(pyaml, new RegExp(`uuid: ${project.openfn.uuid}`));
100+
});
101+
102+
test.todo('fetch throws if writing a new project UUID to an existing file');
103+
104+
test('fetch an existing project with an alias', async (t) => {
105+
const { workspace, read } = initWorkspace(t);
106+
const project = gen();
107+
108+
// fetch the project locally
109+
await run(
110+
`openfn project fetch \
111+
--workspace ${workspace} \
112+
--endpoint ${endpoint} \
113+
--api-key abc \
114+
--alias staging \
115+
${project.openfn.uuid}`
116+
);
117+
118+
const before = read('.projects/staging@localhost.yaml');
119+
t.regex(before, /fn\(\)/);
120+
121+
// now update the remote project
122+
project.workflows[0].steps[0].expression = 'fn(x)';
123+
const state = project.serialize('state', { format: 'json' });
124+
lightning.addProject(state);
125+
126+
// Now run another fetch but only use the alias - no uuid
127+
await run(
128+
`openfn project fetch \
129+
--workspace ${workspace} \
130+
--endpoint ${endpoint} \
131+
--api-key abc \
132+
staging`
133+
);
134+
135+
// now check that the filesystem is roughly right
136+
const after = read('.projects/staging@localhost.yaml');
137+
138+
t.regex(after, /fn\(x\)/);
139+
});
140+
141+
test('pull a new project', async (t) => {
142+
const { workspace, read } = initWorkspace(t);
143+
const project = gen();
144+
145+
await run(
146+
`openfn project pull \
147+
--workspace ${workspace} \
148+
--endpoint ${endpoint} \
149+
--api-key abc \
150+
--log debug \
151+
${project.openfn.uuid}`
152+
);
153+
154+
// now check that the filesystem is roughly right
155+
const proj_yaml = read('.projects/main@localhost.yaml');
156+
157+
t.regex(proj_yaml, /id: patients/);
158+
t.regex(proj_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
159+
160+
const openfn_yaml = read('openfn.yaml');
161+
t.regex(openfn_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
162+
t.regex(openfn_yaml, new RegExp(`endpoint: ${endpoint}`));
163+
164+
const job = read('workflows/workflow/job.js');
165+
t.is(job, 'fn()');
166+
});
167+
168+
test('pull a new project with an alias', async (t) => {
169+
const { workspace, read } = initWorkspace(t);
170+
const project = gen();
171+
172+
await run(
173+
`openfn project pull \
174+
--workspace ${workspace} \
175+
--endpoint ${endpoint} \
176+
--api-key abc \
177+
--log debug \
178+
--alias staging \
179+
${project.openfn.uuid}`
180+
);
181+
182+
// now check that the filesystem is roughly right
183+
const proj_yaml = read('.projects/staging@localhost.yaml');
184+
185+
t.regex(proj_yaml, /id: patients/);
186+
t.regex(proj_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
187+
188+
const openfn_yaml = read('openfn.yaml');
189+
t.regex(openfn_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
190+
t.regex(openfn_yaml, new RegExp(`endpoint: ${endpoint}`));
191+
192+
const job = read('workflows/workflow/job.js');
193+
t.is(job, 'fn()');
194+
});
195+
196+
test('pull an update to project', async (t) => {
197+
const { workspace, read } = initWorkspace(t);
198+
const project = gen();
199+
200+
// fetch the project once to set up the repo
201+
await run(
202+
`openfn project pull \
203+
--workspace ${workspace} \
204+
--endpoint ${endpoint} \
205+
--api-key abc \
206+
${project.openfn.uuid}`
207+
);
208+
209+
const job = read('workflows/workflow/job.js');
210+
t.is(job, 'fn()');
211+
212+
// now update the remote project
213+
project.workflows[0].steps[0].expression = 'fn(x)';
214+
const state = project.serialize('state', { format: 'json' });
215+
lightning.addProject(state);
216+
// (note that the verison hash hasn't updated so not the best test)
217+
218+
// and refetch
219+
await run(
220+
`openfn project pull \
221+
--workspace ${workspace} \
222+
--endpoint ${endpoint} \
223+
--api-key abc \
224+
${project.openfn.uuid}`
225+
);
226+
227+
const proj_yaml = read('.projects/main@localhost.yaml');
228+
t.regex(proj_yaml, /fn\(x\)/);
229+
t.regex(proj_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
230+
231+
const openfn_yaml = read('openfn.yaml');
232+
t.regex(openfn_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
233+
t.regex(openfn_yaml, new RegExp(`endpoint: ${endpoint}`));
234+
235+
const job_updated = read('workflows/workflow/job.js');
236+
t.is(job_updated, 'fn()');
237+
});
238+
239+
test('checkout by alias', async (t) => {
240+
const { workspace, read } = initWorkspace(t);
241+
const main = gen();
242+
const staging = gen('patients-staging', ['trigger-job(body="fn(x)")']);
243+
244+
await run(
245+
`openfn project fetch \
246+
--workspace ${workspace} \
247+
--endpoint ${endpoint} \
248+
--api-key abc \
249+
--alias main\
250+
${main.openfn.uuid}`
251+
);
252+
await run(
253+
`openfn project fetch \
254+
--workspace ${workspace} \
255+
--endpoint ${endpoint} \
256+
--api-key abc \
257+
--alias staging\
258+
${staging.openfn.uuid}`
259+
);
260+
261+
// Ensure the repo is set up correctly
262+
const main_yaml = read('.projects/main@localhost.yaml');
263+
t.regex(main_yaml, /fn\(\)/);
264+
const staging_yaml = read('.projects/staging@localhost.yaml');
265+
t.regex(staging_yaml, /fn\(x\)/);
266+
267+
await run(
268+
`openfn project checkout main \
269+
--workspace ${workspace}`
270+
);
271+
272+
// only do a rough check of the file system
273+
// local tests can be more thorough - at this level
274+
// I just want to see that the command has basically worked
275+
let job = read('workflows/workflow/job.js');
276+
t.is(job, 'fn()');
277+
278+
await run(
279+
`openfn project checkout staging \
280+
--workspace ${workspace}`
281+
);
282+
283+
job = read('workflows/workflow/job.js');
284+
t.is(job, 'fn(x)');
285+
});
286+
287+
test.todo('merge by alias');

‎packages/cli/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
"pnpm": ">=7"
88
},
99
"scripts": {
10-
"test": "pnpm ava",
10+
"test": "pnpm ava --timeout 10m",
1111
"test:watch": "pnpm ava -w",
1212
"test:types": "pnpm tsc --project tsconfig.test.json",
1313
"build": "tsup --config ./tsup.config.js",

‎packages/cli/src/commands.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ export type CommandList =
3939
| 'repo-install'
4040
| 'repo-list'
4141
| 'repo-pwd'
42+
| 'project-pull'
4243
| 'project-list'
4344
| 'project-version'
4445
| 'project-merge'
@@ -66,6 +67,7 @@ const handlers = {
6667
['repo-install']: repo.install,
6768
['repo-pwd']: repo.pwd,
6869
['repo-list']: repo.list,
70+
['project-pull']: projects.pull,
6971
['project-list']: projects.list,
7072
['project-version']: projects.version,
7173
['project-merge']: projects.merge,

‎packages/cli/src/deploy/beta.ts‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,7 @@ import { loadAppAuthConfig } from '../projects/util';
99
export type DeployOptionsBeta = Required<
1010
Pick<
1111
Opts,
12-
| 'beta'
13-
| 'command'
14-
| 'log'
15-
| 'logJson'
16-
| 'apiKey'
17-
| 'endpoint'
18-
| 'path'
19-
| 'workspace'
12+
'beta' | 'command' | 'log' | 'logJson' | 'apiKey' | 'endpoint' | 'path'
2013
>
2114
>;
2215

@@ -25,7 +18,10 @@ export async function handler(options: DeployOptionsBeta, logger: Logger) {
2518

2619
// TMP use options.path to set the directory for now
2720
// We'll need to manage this a bit better
28-
const project = await Project.from('fs', { root: options.workspace || '.' });
21+
// TODO this is fixed on another branch
22+
const project = await Project.from('fs', {
23+
root: (options as any).workspace || '.',
24+
});
2925
// TODO: work out if there's any diff
3026

3127
// generate state for the provisioner

0 commit comments

Comments
 (0)