Skip to content

Commit 07c1510

Browse files
tiny-jamestiny-james
committed
TCLOUD-4780: Generating redirects
1 parent 3a8e729 commit 07c1510

File tree

2 files changed

+3603
-0
lines changed

2 files changed

+3603
-0
lines changed

‎.github/workflows/scripts/generate_redirects.mjs‎

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
import fs from 'node:fs';
2+
import path from 'node:path';
3+
import child_process from 'node:child_process';
4+
5+
6+
/**
7+
* Check if an S3 object exists
8+
*/
9+
function checkS3ObjectExists(_dryRun, _bucket, _prefix, subPath) {
10+
// it's too slow to talk to s3, so just check the local files we just uploaded...
11+
if (subPath.startsWith('docs/')) {
12+
return fs.existsSync(path.join(import.meta.dirname, '../../../build/site', subPath.slice('docs/'.length)));
13+
} else {
14+
return false;
15+
}
16+
}
17+
18+
const metadataArgs = (metadata) => {
19+
// Build metadata string in the format key1=value1,key2=value2
20+
const metadataString = Object.entries(metadata)
21+
.map(([key, value]) => `${key}=${value}`)
22+
.join(',');
23+
24+
return metadataString ? ['--metadata', metadataString] : [];
25+
}
26+
27+
/**
28+
* Copy existing S3 object to itself with new metadata
29+
*/
30+
function copyS3ObjectWithMetadata(dryRun, bucket, prefix, subPath, metadata) {
31+
const fullPath = `${prefix}/${subPath}`;
32+
const cmd = [
33+
'aws', 's3api', 'copy-object',
34+
'--bucket', bucket,
35+
'--copy-source', `${bucket}/${fullPath}`,
36+
'--key', subPath,
37+
'--metadata-directive', 'REPLACE',
38+
'--content-type', 'text/html',
39+
...metadataArgs(metadata)
40+
];
41+
42+
console.log(`Updating existing S3 object with metadata: ${fullPath}`);
43+
console.log(`Command: ${cmd.join(' ')}`);
44+
45+
const result = dryRun ? { status: 0 } : child_process.spawnSync('aws', cmd.slice(1), {
46+
stdio: 'inherit',
47+
encoding: 'utf8'
48+
});
49+
50+
if (result.error) {
51+
console.error(`Error copying S3 object ${fullPath}:`, result.error);
52+
return false;
53+
} else if (result.status !== 0) {
54+
console.error(`AWS CLI copy command failed for ${fullPath} with exit code:`, result.status);
55+
return false;
56+
} else {
57+
console.log(`Successfully updated S3 object metadata: ${fullPath}`);
58+
return true;
59+
}
60+
}
61+
62+
/**
63+
* Create new S3 object with generated content and metadata
64+
*/
65+
function createNewS3Object(dryRun, bucket, prefix, subPath, metadata, newFileTemplate) {
66+
const fullPath = `${prefix}/${subPath}`;
67+
// AWS CLI command to put object with metadata
68+
const cmd = [
69+
'aws', 's3api', 'put-object',
70+
'--bucket', bucket,
71+
'--key', fullPath,
72+
'--body', newFileTemplate,
73+
'--content-type', 'text/html',
74+
...metadataArgs(metadata)
75+
];
76+
77+
console.log(`Creating new S3 object: ${fullPath}`);
78+
console.log(`Command: ${cmd.join(' ')}`);
79+
80+
const result = dryRun ? { status: 0 } : child_process.spawnSync('aws', cmd.slice(1), {
81+
stdio: 'inherit',
82+
encoding: 'utf8'
83+
});
84+
85+
if (result.error) {
86+
console.error(`Error creating S3 object ${fullPath}:`, result.error);
87+
return false;
88+
} else if (result.status !== 0) {
89+
console.error(`AWS CLI command failed for ${fullPath} with exit code:`, result.status);
90+
return false;
91+
} else {
92+
console.log(`Successfully created S3 object: ${fullPath}`);
93+
return true;
94+
}
95+
}
96+
97+
/**
98+
* Create or update S3 object with metadata, reusing existing content if available
99+
*/
100+
function createOrUpdateS3Object(dryRun, bucket, prefix, subPath, metadata, newFileTemplate) {
101+
console.log(`\nProcessing: ${subPath}`);
102+
103+
// Check if object already exists
104+
if (checkS3ObjectExists(dryRun, bucket, prefix, subPath)) {
105+
console.log(`Object exists, updating metadata...`);
106+
return copyS3ObjectWithMetadata(dryRun, bucket, prefix, subPath, metadata);
107+
} else {
108+
console.log(`Object doesn't exist, creating new one...`);
109+
return createNewS3Object(dryRun, bucket, prefix, subPath, metadata, newFileTemplate);
110+
}
111+
}
112+
113+
/**
114+
* Generate S3 objects for all redirects
115+
*/
116+
function generateRedirectObjects(dryRun, bucket, prefix, redirectsByLocation) {
117+
console.log(`Processing ${redirectsByLocation.size} unique locations`);
118+
119+
let successCount = 0;
120+
let errorCount = 0;
121+
122+
// Create empty index.html content for the redirect
123+
const htmlContent = `<!DOCTYPE html>
124+
<html>
125+
<head>
126+
<meta charset="utf-8">
127+
<title>Redirecting...</title>
128+
</head>
129+
<body>
130+
<p>Redirecting...</p>
131+
</body>
132+
</html>`;
133+
134+
// Write temporary file
135+
const newFileTemplate = `/tmp/redirect-${Date.now()}.html`;
136+
fs.writeFileSync(newFileTemplate, htmlContent);
137+
138+
try {
139+
for (const [location, locationRedirects] of redirectsByLocation) {
140+
// Create S3 object path by appending index.html to location
141+
const locationIndexHtml = location.endsWith('/')
142+
? `${location}index.html`
143+
: `${location}/index.html`;
144+
145+
// Remove leading slash from location
146+
const subPath = locationIndexHtml.startsWith('/') ? locationIndexHtml.slice(1) : locationIndexHtml;
147+
148+
// Build metadata headers
149+
const metadata = {};
150+
151+
locationRedirects.forEach((redirect, index) => {
152+
const i = index + 1; // 1-based indexing as requested
153+
154+
// Add redirect location header
155+
metadata[`redirect-location-${i}`] = redirect.redirect;
156+
157+
// Add pattern header if it exists
158+
if (redirect.pattern !== undefined) {
159+
metadata[`redirect-pattern-${i}`] = redirect.pattern;
160+
}
161+
});
162+
163+
// Create or update the S3 object
164+
if (createOrUpdateS3Object(dryRun, bucket, prefix, subPath, metadata, newFileTemplate)) {
165+
successCount++;
166+
} else {
167+
errorCount++;
168+
}
169+
}
170+
} finally {
171+
// Clean up temporary file
172+
if (fs.existsSync(newFileTemplate)) {
173+
fs.unlinkSync(newFileTemplate);
174+
}
175+
}
176+
177+
console.log(`\nSummary:`);
178+
console.log(`Successfully processed: ${successCount} objects`);
179+
console.log(`Errors: ${errorCount} objects`);
180+
}
181+
182+
183+
const usage = () => `
184+
generate-redirects [--dry-run] <bucket> <prefix>
185+
Generate redirects in s3.
186+
187+
Options:
188+
--dry-run only output the commands that will be run
189+
`
190+
191+
const main = async () => {
192+
193+
const args = process.argv.slice(2);
194+
const dryRun = (() => {
195+
const idx = args.findIndex((arg) => arg === '--dry-run');
196+
if (idx !== -1) {
197+
args.splice(idx, 1);
198+
return true;
199+
}
200+
return false;
201+
})();
202+
if (args.length !== 2) {
203+
return Promise.reject(`Expected 2 values, got ${args.length}`);
204+
}
205+
const [bucket, prefix] = args;
206+
if (!/^[a-z0-9][a-z0-9\.-]{1,61}[a-z0-9]$/.test(bucket) ||
207+
/\.\./.test(bucket) || /^\d+\.\d+\.\d+\.\d+$/.test(bucket) ||
208+
/^xn--/.test(bucket) || /^sthree-/.test(bucket) || /^amzn-s3-demo-/.test(bucket) ||
209+
/-s3alias$/.test(bucket) || /--ol-s3$/.test(bucket) || /\.mrap$/.test(bucket) ||
210+
/--x-s3$/.test(bucket) || /--table-s3$/.test(bucket)) {
211+
return Promise.reject(`Invalid bucket name, got ${bucket}`);
212+
}
213+
214+
if (!/^[a-z0-9\.-]+$/.test(prefix)) {
215+
return Promise.reject(`Invalid prefix, got ${prefix}`);
216+
}
217+
218+
const redirects = JSON.parse(fs.readFileSync(path.join(import.meta.dirname, '../../../redirects.json'), 'utf-8'));
219+
220+
// Group redirects by location to handle multiple redirects for the same location
221+
const redirectsByLocation = new Map();
222+
223+
redirects.forEach((redirect) => {
224+
const location = redirect.location;
225+
if (!redirectsByLocation.has(location)) {
226+
redirectsByLocation.set(location, []);
227+
}
228+
redirectsByLocation.get(location).push(redirect);
229+
});
230+
231+
// Generate all redirect objects
232+
generateRedirectObjects(dryRun, bucket, prefix, redirectsByLocation);
233+
234+
console.log('Redirect object generation completed.');
235+
};
236+
237+
238+
main().catch((err) => {
239+
console.error(err);
240+
console.error(usage());
241+
process.exit(1);
242+
})

0 commit comments

Comments
 (0)