Skip to content

Commit 161de11

Browse files
netroynetroy
authored
fix(core): Improve community nodes loading (#5608)
1 parent 970c124 commit 161de11

File tree

9 files changed

+74
-138
lines changed

9 files changed

+74
-138
lines changed

‎packages/cli/src/CredentialsHelper.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -451,7 +451,6 @@ export class CredentialsHelper extends ICredentialsHelper {
451451
type: string,
452452
data: ICredentialDataDecryptedObject,
453453
): Promise<void> {
454-
// eslint-disable-next-line @typescript-eslint/await-thenable
455454
const credentials = await this.getCredentials(nodeCredentials, type);
456455

457456
if (!Db.isInitialized) {

‎packages/cli/src/LoadNodesAndCredentials.ts‎

Lines changed: 61 additions & 130 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import uniq from 'lodash.uniq';
2+
import glob from 'fast-glob';
23
import type { DirectoryLoader, Types } from 'n8n-core';
34
import {
45
CUSTOM_EXTENSION_ENV,
@@ -18,18 +19,18 @@ import type {
1819
import { LoggerProxy, ErrorReporterProxy as ErrorReporter } from 'n8n-workflow';
1920

2021
import { createWriteStream } from 'fs';
21-
import { access as fsAccess, mkdir, readdir as fsReaddir, stat as fsStat } from 'fs/promises';
22+
import { mkdir } from 'fs/promises';
2223
import path from 'path';
2324
import config from '@/config';
2425
import type { InstalledPackages } from '@db/entities/InstalledPackages';
2526
import { executeCommand } from '@/CommunityNodes/helpers';
2627
import {
27-
CLI_DIR,
2828
GENERATED_STATIC_DIR,
2929
RESPONSE_ERROR_MESSAGES,
3030
CUSTOM_API_CALL_KEY,
3131
CUSTOM_API_CALL_NAME,
3232
inTest,
33+
CLI_DIR,
3334
} from '@/constants';
3435
import { CredentialsOverwrites } from '@/CredentialsOverwrites';
3536
import { Service } from 'typedi';
@@ -52,6 +53,8 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
5253

5354
logger: ILogger;
5455

56+
private downloadFolder: string;
57+
5558
async init() {
5659
// Make sure the imported modules can resolve dependencies fine.
5760
const delimiter = process.platform === 'win32' ? ';' : ':';
@@ -61,8 +64,13 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
6164
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
6265
if (!inTest) module.constructor._initPaths();
6366

64-
await this.loadNodesFromBasePackages();
65-
await this.loadNodesFromDownloadedPackages();
67+
this.downloadFolder = UserSettings.getUserN8nFolderDownloadedNodesPath();
68+
69+
// Load nodes from `n8n-nodes-base` and any other `n8n-nodes-*` package in the main `node_modules`
70+
await this.loadNodesFromNodeModules(CLI_DIR);
71+
// Load nodes from installed community packages
72+
await this.loadNodesFromNodeModules(this.downloadFolder);
73+
6674
await this.loadNodesFromCustomDirectories();
6775
await this.postProcessLoaders();
6876
this.injectCustomApiCallOptions();
@@ -109,32 +117,20 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
109117
await writeStaticJSON('credentials', this.types.credentials);
110118
}
111119

112-
private async loadNodesFromBasePackages() {
113-
const nodeModulesPath = await this.getNodeModulesPath();
114-
const nodePackagePaths = await this.getN8nNodePackages(nodeModulesPath);
115-
116-
for (const packagePath of nodePackagePaths) {
117-
await this.runDirectoryLoader(LazyPackageDirectoryLoader, packagePath);
118-
}
119-
}
120-
121-
private async loadNodesFromDownloadedPackages(): Promise<void> {
122-
const nodePackages = [];
123-
try {
124-
// Read downloaded nodes and credentials
125-
const downloadedNodesDir = UserSettings.getUserN8nFolderDownloadedNodesPath();
126-
const downloadedNodesDirModules = path.join(downloadedNodesDir, 'node_modules');
127-
await fsAccess(downloadedNodesDirModules);
128-
const downloadedPackages = await this.getN8nNodePackages(downloadedNodesDirModules);
129-
nodePackages.push(...downloadedPackages);
130-
} catch (error) {
131-
// Folder does not exist so ignore and return
132-
return;
133-
}
120+
private async loadNodesFromNodeModules(scanDir: string): Promise<void> {
121+
const nodeModulesDir = path.join(scanDir, 'node_modules');
122+
const globOptions = { cwd: nodeModulesDir, onlyDirectories: true };
123+
const installedPackagePaths = [
124+
...(await glob('n8n-nodes-*', { ...globOptions, deep: 1 })),
125+
...(await glob('@*/n8n-nodes-*', { ...globOptions, deep: 2 })),
126+
];
134127

135-
for (const packagePath of nodePackages) {
128+
for (const packagePath of installedPackagePaths) {
136129
try {
137-
await this.runDirectoryLoader(PackageDirectoryLoader, packagePath);
130+
await this.runDirectoryLoader(
131+
LazyPackageDirectoryLoader,
132+
path.join(nodeModulesDir, packagePath),
133+
);
138134
} catch (error) {
139135
ErrorReporter.error(error);
140136
}
@@ -158,49 +154,45 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
158154
}
159155
}
160156

161-
/**
162-
* Returns all the names of the packages which could contain n8n nodes
163-
*/
164-
private async getN8nNodePackages(baseModulesPath: string): Promise<string[]> {
165-
const getN8nNodePackagesRecursive = async (relativePath: string): Promise<string[]> => {
166-
const results: string[] = [];
167-
const nodeModulesPath = `${baseModulesPath}/${relativePath}`;
168-
const nodeModules = await fsReaddir(nodeModulesPath);
169-
for (const nodeModule of nodeModules) {
170-
const isN8nNodesPackage = nodeModule.indexOf('n8n-nodes-') === 0;
171-
const isNpmScopedPackage = nodeModule.indexOf('@') === 0;
172-
if (!isN8nNodesPackage && !isNpmScopedPackage) {
173-
continue;
174-
}
175-
if (!(await fsStat(nodeModulesPath)).isDirectory()) {
176-
continue;
177-
}
178-
if (isN8nNodesPackage) {
179-
results.push(`${baseModulesPath}/${relativePath}${nodeModule}`);
180-
}
181-
if (isNpmScopedPackage) {
182-
results.push(...(await getN8nNodePackagesRecursive(`${relativePath}${nodeModule}/`)));
183-
}
184-
}
185-
return results;
186-
};
187-
return getN8nNodePackagesRecursive('');
188-
}
189-
190-
async loadNpmModule(packageName: string, version?: string): Promise<InstalledPackages> {
191-
const downloadFolder = UserSettings.getUserN8nFolderDownloadedNodesPath();
192-
const command = `npm install ${packageName}${version ? `@${version}` : ''}`;
157+
private async installOrUpdateNpmModule(
158+
packageName: string,
159+
options: { version?: string } | { installedPackage: InstalledPackages },
160+
) {
161+
const isUpdate = 'installedPackage' in options;
162+
const command = isUpdate
163+
? `npm update ${packageName}`
164+
: `npm install ${packageName}${options.version ? `@${options.version}` : ''}`;
193165

194-
await executeCommand(command);
166+
try {
167+
await executeCommand(command);
168+
} catch (error) {
169+
if (error instanceof Error && error.message === RESPONSE_ERROR_MESSAGES.PACKAGE_NOT_FOUND) {
170+
throw new Error(`The npm package "${packageName}" could not be found.`);
171+
}
172+
throw error;
173+
}
195174

196-
const finalNodeUnpackedPath = path.join(downloadFolder, 'node_modules', packageName);
175+
const finalNodeUnpackedPath = path.join(this.downloadFolder, 'node_modules', packageName);
197176

198-
const loader = await this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath);
177+
let loader: PackageDirectoryLoader;
178+
try {
179+
loader = await this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath);
180+
} catch (error) {
181+
// Remove this package since loading it failed
182+
const removeCommand = `npm remove ${packageName}`;
183+
try {
184+
await executeCommand(removeCommand);
185+
} catch {}
186+
throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_LOADING_FAILED, { cause: error });
187+
}
199188

200189
if (loader.loadedNodes.length > 0) {
201190
// Save info to DB
202191
try {
203-
const { persistInstalledPackageData } = await import('@/CommunityNodes/packageModel');
192+
const { persistInstalledPackageData, removePackageFromDatabase } = await import(
193+
'@/CommunityNodes/packageModel'
194+
);
195+
if (isUpdate) await removePackageFromDatabase(options.installedPackage);
204196
const installedPackage = await persistInstalledPackageData(loader);
205197
await this.postProcessLoaders();
206198
await this.generateTypesForFrontend();
@@ -223,6 +215,10 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
223215
}
224216
}
225217

218+
async installNpmModule(packageName: string, version?: string): Promise<InstalledPackages> {
219+
return this.installOrUpdateNpmModule(packageName, { version });
220+
}
221+
226222
async removeNpmModule(packageName: string, installedPackage: InstalledPackages): Promise<void> {
227223
const command = `npm remove ${packageName}`;
228224

@@ -244,49 +240,7 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
244240
packageName: string,
245241
installedPackage: InstalledPackages,
246242
): Promise<InstalledPackages> {
247-
const downloadFolder = UserSettings.getUserN8nFolderDownloadedNodesPath();
248-
249-
const command = `npm i ${packageName}@latest`;
250-
251-
try {
252-
await executeCommand(command);
253-
} catch (error) {
254-
if (error instanceof Error && error.message === RESPONSE_ERROR_MESSAGES.PACKAGE_NOT_FOUND) {
255-
throw new Error(`The npm package "${packageName}" could not be found.`);
256-
}
257-
throw error;
258-
}
259-
260-
const finalNodeUnpackedPath = path.join(downloadFolder, 'node_modules', packageName);
261-
262-
const loader = await this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath);
263-
264-
if (loader.loadedNodes.length > 0) {
265-
// Save info to DB
266-
try {
267-
const { persistInstalledPackageData, removePackageFromDatabase } = await import(
268-
'@/CommunityNodes/packageModel'
269-
);
270-
await removePackageFromDatabase(installedPackage);
271-
const newlyInstalledPackage = await persistInstalledPackageData(loader);
272-
await this.postProcessLoaders();
273-
await this.generateTypesForFrontend();
274-
return newlyInstalledPackage;
275-
} catch (error) {
276-
LoggerProxy.error('Failed to save installed packages and nodes', {
277-
error: error as Error,
278-
packageName,
279-
});
280-
throw error;
281-
}
282-
} else {
283-
// Remove this package since it contains no loadable nodes
284-
const removeCommand = `npm remove ${packageName}`;
285-
try {
286-
await executeCommand(removeCommand);
287-
} catch {}
288-
throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_DOES_NOT_CONTAIN_NODES);
289-
}
243+
return this.installOrUpdateNpmModule(packageName, { installedPackage });
290244
}
291245

292246
/**
@@ -399,27 +353,4 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
399353
}
400354
}
401355
}
402-
403-
private async getNodeModulesPath(): Promise<string> {
404-
// Get the path to the node-modules folder to be later able
405-
// to load the credentials and nodes
406-
const checkPaths = [
407-
// In case "n8n" package is in same node_modules folder.
408-
path.join(CLI_DIR, '..', 'n8n-workflow'),
409-
// In case "n8n" package is the root and the packages are
410-
// in the "node_modules" folder underneath it.
411-
path.join(CLI_DIR, 'node_modules', 'n8n-workflow'),
412-
// In case "n8n" package is installed using npm/yarn workspaces
413-
// the node_modules folder is in the root of the workspace.
414-
path.join(CLI_DIR, '..', '..', 'node_modules', 'n8n-workflow'),
415-
];
416-
for (const checkPath of checkPaths) {
417-
try {
418-
await fsAccess(checkPath);
419-
// Folder exists, so use it.
420-
return path.dirname(checkPath);
421-
} catch {} // Folder does not exist so get next one
422-
}
423-
throw new Error('Could not find "node_modules" folder!');
424-
}
425356
}

‎packages/cli/src/commands/start.ts‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -267,11 +267,10 @@ export class Start extends BaseCommand {
267267
// Optimistic approach - stop if any installation fails
268268
// eslint-disable-next-line no-restricted-syntax
269269
for (const missingPackage of missingPackages) {
270-
// eslint-disable-next-line no-await-in-loop
271-
void (await this.loadNodesAndCredentials.loadNpmModule(
270+
await this.loadNodesAndCredentials.installNpmModule(
272271
missingPackage.packageName,
273272
missingPackage.version,
274-
));
273+
);
275274
missingPackages.delete(missingPackage);
276275
}
277276
LoggerProxy.info('Packages reinstalled successfully. Resuming regular initialization.');

‎packages/cli/src/constants.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ export const CUSTOM_API_CALL_KEY = '__CUSTOM_API_CALL__';
1818

1919
export const CLI_DIR = resolve(__dirname, '..');
2020
export const TEMPLATES_DIR = join(CLI_DIR, 'templates');
21-
export const NODES_BASE_DIR = join(CLI_DIR, '..', 'nodes-base');
21+
export const NODES_BASE_DIR = dirname(require.resolve('n8n-nodes-base'));
2222
export const GENERATED_STATIC_DIR = join(UserSettings.getUserHome(), '.cache/n8n/public');
2323
export const EDITOR_UI_DIST_DIR = join(dirname(require.resolve('n8n-editor-ui')), 'dist');
2424

@@ -45,6 +45,7 @@ export const RESPONSE_ERROR_MESSAGES = {
4545
PACKAGE_NOT_FOUND: 'Package not found in npm',
4646
PACKAGE_VERSION_NOT_FOUND: 'The specified package version was not found',
4747
PACKAGE_DOES_NOT_CONTAIN_NODES: 'The specified package does not contain any nodes',
48+
PACKAGE_LOADING_FAILED: 'The specified package could not be loaded',
4849
DISK_IS_FULL: 'There appears to be insufficient disk space',
4950
};
5051

‎packages/cli/src/controllers/nodes.controller.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ export class NodesController {
109109

110110
let installedPackage: InstalledPackages;
111111
try {
112-
installedPackage = await this.loadNodesAndCredentials.loadNpmModule(
112+
installedPackage = await this.loadNodesAndCredentials.installNpmModule(
113113
parsed.packageName,
114114
parsed.version,
115115
);
@@ -125,7 +125,10 @@ export class NodesController {
125125
failure_reason: errorMessage,
126126
});
127127

128-
const message = [`Error loading package "${name}"`, errorMessage].join(':');
128+
let message = [`Error loading package "${name}" `, errorMessage].join(':');
129+
if (error instanceof Error && error.cause instanceof Error) {
130+
message += `\nCause: ${error.cause.message}`;
131+
}
129132

130133
const clientError = error instanceof Error ? isClientError(error) : false;
131134
throw new (clientError ? BadRequestError : InternalServerError)(message);

‎packages/cli/test/integration/nodes.api.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ describe('POST /nodes', () => {
191191
mocked(hasPackageLoaded).mockReturnValueOnce(false);
192192
mocked(checkNpmPackageStatus).mockResolvedValueOnce({ status: 'OK' });
193193

194-
mockLoadNodesAndCredentials.loadNpmModule.mockImplementationOnce(mockedEmptyPackage);
194+
mockLoadNodesAndCredentials.installNpmModule.mockImplementationOnce(mockedEmptyPackage);
195195

196196
const { statusCode } = await authOwnerShellAgent.post('/nodes').send({
197197
name: utils.installedPackagePayload().packageName,

‎packages/nodes-base/.eslintrc.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ module.exports = {
88

99
...sharedOptions(__dirname),
1010

11+
ignorePatterns: ['index.js'],
12+
1113
rules: {
1214
'@typescript-eslint/consistent-type-imports': 'error',
1315

‎packages/nodes-base/index.js‎

Whitespace-only changes.

‎packages/nodes-base/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
"name": "Jan Oberhauser",
99
"email": "jan@n8n.io"
1010
},
11+
"main": "index.js",
1112
"repository": {
1213
"type": "git",
1314
"url": "git+https://github.com/n8n-io/n8n.git"

0 commit comments

Comments
 (0)