Skip to content

Commit 5236e30

Browse files
authored
fix(s3-upload): Dont include all built (#416)
## Describe your changes ## Issue ticket number and link ## Checklist before requesting a review (skip if just adding/editing APIs & templates) - [ ] I added tests, otherwise the reason is: - [ ] External API requests have `retries` - [ ] Pagination is used where appropriate - [ ] The built in `nango.paginate` call is used instead of a `while (true)` loop - [ ] Third party requests are NOT parallelized (this can cause issues with rate limits) - [ ] If a sync requires metadata the `nango.yaml` has `auto_start: false` - [ ] If the sync is a `full` sync then `track_deletes: true` is set - [ ] I followed the best practices and guidelines from the [Writing Integration Scripts](/NangoHQ/integration-templates/blob/main/guides/WRITING_SCRIPTS.md) doc
1 parent c0269d1 commit 5236e30

File tree

2 files changed

+65
-3
lines changed

2 files changed

+65
-3
lines changed

.github/workflows/deploy.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,5 +77,5 @@ jobs:
7777
rm -rf integrations/build
7878
npm run compile:integrations
7979
aws s3 sync $dir s3://${{ secrets.AWS_BUCKET_NAME }}/templates-zero/$integration
80-
aws s3 sync integrations/build s3://${{ secrets.AWS_BUCKET_NAME }}/templates-zero/$integration/build
80+
aws s3 sync integrations/$integration/build s3://${{ secrets.AWS_BUCKET_NAME }}/templates-zero/$integration/build
8181
done

internal/scripts/zero/compileIntegrationsJson.ts

Lines changed: 64 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,11 @@
55
/* eslint-disable @nangohq/custom-integrations-linting/no-console-log */
66
/* eslint-disable @nangohq/custom-integrations-linting/no-try-catch-unless-explicitly-allowed */
77

8-
import { readFile, writeFile, readdir, lstat, readlink } from 'fs/promises';
8+
import { readFile, writeFile, readdir, lstat, readlink, mkdir, copyFile, rm } from 'fs/promises';
99
import { join, basename, dirname } from 'path';
1010
import { execSync } from 'child_process';
1111
import { fileURLToPath } from 'url';
12+
import { existsSync } from 'fs';
1213
import type { NangoYamlParsedIntegration } from '@nangohq/types';
1314
import chalk from 'chalk';
1415
import { errorToString } from './utils.js';
@@ -110,10 +111,71 @@ async function main(): Promise<void> {
110111
process.exit(1);
111112
}
112113

114+
// Step 2.5: Distribute build files to each integration's build directory
115+
console.log();
116+
console.log('Distributing build files to integration directories...');
117+
118+
const centralBuildDir = join(integrationsPath, 'build');
119+
if (!existsSync(centralBuildDir)) {
120+
console.log(` ${chalk.yellow('warn')} No build directory found, skipping distribution`);
121+
} else {
122+
// Read all .cjs files from the central build directory
123+
const buildFiles = await readdir(centralBuildDir);
124+
const cjsFiles = buildFiles.filter((f) => f.endsWith('.cjs'));
125+
126+
// Group files by integration name (prefix before first underscore)
127+
const filesByIntegration = new Map<string, string[]>();
128+
for (const file of cjsFiles) {
129+
// Files are named like: airtable_syncs_bases.cjs or airtable_actions_create-webhook.cjs
130+
const underscoreIndex = file.indexOf('_');
131+
if (underscoreIndex === -1) continue;
132+
133+
const integrationName = file.substring(0, underscoreIndex);
134+
if (!filesByIntegration.has(integrationName)) {
135+
filesByIntegration.set(integrationName, []);
136+
}
137+
filesByIntegration.get(integrationName)!.push(file);
138+
}
139+
140+
// Distribute files to each integration's build directory
141+
for (const [integrationName, files] of filesByIntegration) {
142+
const integrationDir = join(integrationsPath, integrationName);
143+
144+
// Skip if integration directory doesn't exist
145+
if (!existsSync(integrationDir)) {
146+
continue;
147+
}
148+
149+
// Skip symlinked directories to avoid overwriting the target's build
150+
const stat = await lstat(integrationDir);
151+
if (stat.isSymbolicLink()) {
152+
continue;
153+
}
154+
155+
const integrationBuildDir = join(integrationDir, 'build');
156+
157+
// Clear and recreate build directory
158+
if (existsSync(integrationBuildDir)) {
159+
await rm(integrationBuildDir, { recursive: true });
160+
}
161+
await mkdir(integrationBuildDir, { recursive: true });
162+
163+
// Copy each file, keeping the original filename
164+
for (const file of files) {
165+
const sourceFile = join(centralBuildDir, file);
166+
const destFile = join(integrationBuildDir, file);
167+
await copyFile(sourceFile, destFile);
168+
}
169+
170+
console.log(` ${chalk.green('✓')} ${integrationName} (${files.length} files)`);
171+
}
172+
}
173+
174+
const fullSchema = jsonSchema as { $schema?: string; $comment?: string; definitions: Record<string, unknown> };
175+
113176
// Step 3: Transform to ZeroFlow format
114177
// Each integration in nangoData already has providerConfigKey set
115178
// Filter jsonSchema to only include models used by each integration
116-
const fullSchema = jsonSchema as { $schema?: string; $comment?: string; definitions: Record<string, unknown> };
117179

118180
const aggregatedFlows: ZeroFlow[] = nangoData.map((integration) => {
119181
// Collect all used models from syncs and actions

0 commit comments

Comments
 (0)