Skip to content
This repository was archived by the owner on Aug 6, 2025. It is now read-only.

Commit 7245386

Browse files
GuruPKKmadelinezeccasthewiz
authored
DOP-2152 Implement instantly see changes to docs (#439)
* Dop 1986: Instantly see changes on docs on /deploy (#422) * testing * testing * testing * save changes * test * commit before switch branch * commit before switch * temp * test * test * test * 5-11-21 Release version bump (#417) * clean up * clean * clean up * clean * clean * clean * lint * refactor * refactor * refactor * clean * add axios Co-authored-by: Cassidy Schaufele <[email protected]> Co-authored-by: Kumaraguru Periyasamy Kanthasamy <[email protected]> * pointing to right meta file * Update githubJob.js * Update Dockerfile * Update Dockerfile * Update Dockerfile * Cache purge changes (#435) * Dop 1986: Instantly see changes on docs on /deploy (#422) * testing * testing * testing * save changes * test * commit before switch branch * commit before switch * temp * test * test * test * 5-11-21 Release version bump (#417) * clean up * clean * clean up * clean * clean * clean * lint * refactor * refactor * refactor * clean * add axios Co-authored-by: Cassidy Schaufele <[email protected]> Co-authored-by: Kumaraguru Periyasamy Kanthasamy <[email protected]> * pointing to right meta file * Update githubJob.js * Update Dockerfile * Update Dockerfile Co-authored-by: madelinezec <[email protected]> Co-authored-by: Cassidy Schaufele <[email protected]> * Update S3Publish.js * Update S3Publish.js * Update fastlyJob.js * Update S3Publish.js Co-authored-by: madelinezec <[email protected]> Co-authored-by: Cassidy Schaufele <[email protected]>
1 parent 71fed30 commit 7245386

File tree

6 files changed

+202
-130
lines changed

6 files changed

+202
-130
lines changed

‎worker/jobTypes/S3Publish.js‎

Lines changed: 28 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ class S3PublishClass {
8080
//as defined in githubJob.js
8181
if (manifestPrefix) deployCommands[deployCommands.length - 1] += ` MANIFEST_PREFIX=${manifestPrefix} GLOBAL_SEARCH_FLAG=${this.GitHubJob.currentJob.payload.stableBranch}`;
8282
}
83+
8384
// deploy site
8485
try {
8586
const exec = workerUtils.getExecPromise();
@@ -93,57 +94,39 @@ class S3PublishClass {
9394
);
9495
throw new Error(`Failed pushing to prod: ${stderr}`)
9596
}
96-
// check for json string output from mut
97-
const validateJsonOutput = stdout ? stdout.substr(0, stdout.lastIndexOf(']}') + 2) : '';
9897

99-
// check if json was returned from mut
10098
try {
101-
const stdoutJSON = JSON.parse(validateJsonOutput);
102-
const urls = stdoutJSON.urls;
103-
// pass in urls to fastly function to purge cache
104-
this.fastly.purgeCache(urls).then(function (data) {
105-
logger.save(`${'(prod)'.padEnd(15)}Fastly finished purging URL's`);
106-
logger.sendSlackMsg('Fastly Summary: The following pages were purged from cache for your deploy');
107-
// when finished purging
108-
// batch urls to send as single slack message
109-
let batchedUrls = [];
110-
for (let i = 0; i < urls.length; i++) {
111-
const purgedUrl = urls[i];
112-
if (purgedUrl && purgedUrl.indexOf('.html') !== -1) {
113-
batchedUrls.push(purgedUrl);
114-
}
115-
// if over certain length, send as a single slack message and reset the array
116-
if (batchedUrls.length > 20 || i >= (urls.length - 1)) {
117-
logger.sendSlackMsg(`${batchedUrls.join('\n')}`);
118-
batchedUrls = [];
119-
}
120-
}
121-
});
122-
} catch (e) {
123-
// if not JSON, then it's a normal string output from mut
124-
// get only last part of message which includes # of files changes + s3 link
125-
if (stdout.indexOf('Summary') !== -1) {
126-
stdoutMod = stdout.substr(stdout.indexOf('Summary'));
127-
}
128-
}
99+
const makefileOutput = stdout.replace(/\r/g, '').split(/\n/);
100+
// the URLS are always third line returned bc of the makefile target
101+
const stdoutJSON = JSON.parse(makefileOutput[2]);
102+
//contains URLs corresponding to files updated via our push to S3
103+
const updatedURLsArray = stdoutJSON.urls;
104+
// purgeCache purges the now stale content and requests the URLs to warm the cache for our users
105+
logger.save(`${JSON.stringify(updatedURLsArray)}`);
106+
await this.fastly.purgeCache(updatedURLsArray);
107+
//save purged URLs to job object
108+
await workerUtils.updateJobWithPurgedURLs(this.GitHubJob.currentJob, updatedURLsArray);
129109

130-
return new Promise((resolve) => {
131-
logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`);
132-
logger.save(
133-
`${'(prod)'.padEnd(15)}Deploy details:\n\n${stdoutMod}`
134-
);
135-
resolve({
136-
status: 'success',
137-
stdout: stdoutMod
138-
});
139-
});
140-
} catch (errResult) {
141-
logger.save(`${'(prod)'.padEnd(15)}stdErr: ${errResult.stderr}`);
142-
throw errResult;
110+
return new Promise((resolve) => {
111+
logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`);
112+
logger.save(`${'(prod)'.padEnd(15)}Deploy details:\n\n${stdoutMod}`);
113+
resolve({
114+
status: 'success',
115+
stdout: stdoutMod,
116+
});
117+
},);
118+
} catch (error) {
119+
console.trace(error)
120+
throw(error)
143121
}
122+
}
123+
catch (errResult) {
124+
logger.save(`${'(prod)'.padEnd(15)}stdErr: ${errResult.stderr}`);
125+
throw errResult;
126+
}
144127
}
145128
}
146129

147130
module.exports = {
148-
S3PublishClass
131+
S3PublishClass,
149132
};

‎worker/package-lock.json‎

Lines changed: 32 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎worker/package.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
"@babel/polyfill": "^7.4.4",
2020
"async-retry": "^1.2.3",
2121
"aws-sdk": "^2.383.0",
22+
"axios": "^0.21.1",
2223
"core-js": "^3.1.4",
2324
"express": "^4.16.4",
2425
"fastly": "^2.2.1",
@@ -34,10 +35,10 @@
3435
"remote-file-size": "^3.0.5",
3536
"simple-git": "^1.107.0",
3637
"supertest": "^4.0.2",
38+
"toml": "^3.0.0",
3739
"typescript": "^4.0.2",
3840
"validator": "^10.11.0",
39-
"xmlhttprequest": "^1.8.0",
40-
"toml": "^3.0.0"
41+
"xmlhttprequest": "^1.8.0"
4142
},
4243
"devDependencies": {
4344
"aws-sdk-mock": "^4.3.0",

‎worker/utils/fastlyJob.js‎

Lines changed: 115 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -1,92 +1,128 @@
1-
const request = require('request');
2-
const utils = require('../utils/utils');
1+
const axios = require('axios').default;
32
const environment = require('../utils/environment').EnvironmentClass;
43
const fastly = require('fastly')(environment.getFastlyToken());
4+
const utils = require('../utils/utils');
5+
const Logger = require('../utils/logger').LoggerClass;
6+
7+
const fastlyServiceId = environment.getFastlyServiceId();
8+
const headers = {
9+
'Fastly-Key': environment.getFastlyToken(),
10+
'Accept': 'application/json',
11+
'Content-Type': 'application/json',
12+
'Fastly-Debug': 1,
13+
};
14+
515

616
class FastlyJobClass {
7-
// pass in a job payload to setup class
8-
constructor(currentJob) {
9-
this.currentJob = currentJob;
10-
if (fastly === undefined) {
11-
utils.logInMongo(currentJob, 'fastly connectivity not found');
17+
// pass in a job payload to setup class
18+
constructor(currentJob) {
19+
this.currentJob = currentJob;
20+
this.logger = new Logger(currentJob);
21+
if (fastly === undefined) {
22+
utils.logInMongo(currentJob, 'fastly connectivity not found');
23+
}
1224
}
13-
}
1425

15-
// takes in an array of urls and purges cache for each
16-
async purgeCache(urlArray) {
17-
if (!Array.isArray(urlArray)) {
18-
throw new Error('Parameter `urlArray` needs to be an array of urls');
26+
// takes in an array of surrogate keys and purges cache for each
27+
async purgeCache(urlArray) {
28+
if (!Array.isArray(urlArray)) {
29+
throw new Error('Parameter `urlArray` needs to be an array of urls');
30+
}
31+
32+
try {
33+
//retrieve surrogate key associated with each URL/file updated in push to S3
34+
const surrogateKeyPromises = urlArray.map(url => this.retrieveSurrogateKey(url));
35+
const surrogateKeyArray = await Promise.all(surrogateKeyPromises)
36+
37+
//purge each surrogate key
38+
const purgeRequestPromises = surrogateKeyArray.map(surrogateKey => this.requestPurgeOfSurrogateKey(surrogateKey));
39+
await Promise.all(purgeRequestPromises);
40+
41+
// GET request the URLs to warm cache for our users
42+
const warmCachePromises = urlArray.map(url => this.warmCache(url));
43+
await Promise.all(warmCachePromises)
44+
} catch (error) {
45+
this.logger.save(`${'(prod)'.padEnd(15)}error in purge cache: ${error}`);
46+
// throw error
47+
}
48+
1949
}
2050

21-
let that = this;
22-
let urlCounter = urlArray.length;
23-
let purgeMessages = [];
24-
25-
// the 1 is just "some" value needed for this header: https://docs.fastly.com/en/guides/soft-purges
26-
const headers = {
27-
'fastly-key': environment.getFastlyToken(),
28-
'accept': 'application/json',
29-
'Fastly-Soft-Purge': '1',
30-
};
31-
32-
return new Promise((resolve, reject) => {
33-
for (let i = 0; i < urlArray.length; i++) {
34-
// perform request to purge
35-
request({
36-
method: 'PURGE',
37-
url: urlArray[i],
38-
headers: headers,
39-
}, function(err, response, body) {
40-
// url was not valid to purge
41-
if (!response) {
42-
utils.logInMongo(that.currentJob, `Error: service for this url does not exist in fastly for purging ${urlArray[i]}`);
43-
purgeMessages.push({
44-
'status': 'failure',
45-
'message': `service with url ${urlArray[i]} does not exist in fastly`
51+
async retrieveSurrogateKey(url) {
52+
53+
try {
54+
return axios({
55+
method: 'HEAD',
56+
url: url,
57+
headers: headers,
58+
}).then(response => {
59+
if (response.status === 200) {
60+
return response.headers['surrogate-key'];
61+
}
4662
});
47-
} else if (response.headers['content-type'].indexOf('application/json') === 0) {
48-
try {
49-
body = JSON.parse(body);
50-
purgeMessages.push(body);
51-
} catch(er) {
52-
utils.logInMongo(that.currentJob, `Error: failed parsing output from fastly for url ${urlArray[i]}`);
53-
console.log(`Error: failed parsing output from fastly for url ${urlArray[i]}`);
54-
}
55-
}
56-
// when we are done purging all urls
57-
// this is outside of the conditional above because if some url fails to purge
58-
// we do not want to actually have this entire build fail, just show warning
59-
urlCounter--;
60-
if (urlCounter <= 0) {
61-
resolve({
62-
'status': 'success',
63-
'fastlyMessages': purgeMessages,
63+
} catch (error) {
64+
this.logger.save(`${'(prod)'.padEnd(15)}error in retrieveSurrogateKey: ${error}`);
65+
throw error
66+
}
67+
68+
}
69+
70+
async requestPurgeOfSurrogateKey(surrogateKey) {
71+
headers['Surrogate-Key'] = surrogateKey
72+
73+
try {
74+
return axios({
75+
method: `POST`,
76+
url: `https://api.fastly.com/service/${fastlyServiceId}/purge/${surrogateKey}`,
77+
path: `/service/${fastlyServiceId}/purge${surrogateKey}`,
78+
headers: headers,
79+
})
80+
.then(response => {
81+
if (response.status === 200) {
82+
return true
83+
}
84+
});
85+
} catch (error) {
86+
this.logger.save(`${'(prod)'.padEnd(15)}error in requestPurgeOfSurrogateKey: ${error}`);
87+
throw error;
88+
}
89+
}
90+
91+
// request urls of updated content to "warm" the cache for our customers
92+
async warmCache(updatedUrl) {
93+
94+
try {
95+
return axios.get(updatedUrl)
96+
.then(response => {
97+
if (response.status === 200) {
98+
return true;
99+
}
100+
})
101+
} catch (error) {
102+
this.logger.save(`${'(prod)'.padEnd(15)}stdErr: ${error}`);
103+
throw error;
104+
}
105+
}
106+
107+
// upserts {source: target} mappings
108+
// to the fastly edge dictionary
109+
async connectAndUpsert(map) {
110+
const options = {
111+
item_value: map.target,
112+
};
113+
const connectString = `/service/${fastlyServiceId}/dictionary/${environment.getDochubMap()}/item/${
114+
map.source
115+
}`;
116+
117+
return new Promise((resolve, reject) => {
118+
fastly.request('PUT', connectString, options, (err, obj) => {
119+
if (err) reject(err);
120+
resolve(obj);
64121
});
65-
}
66-
});
67-
}
68-
})
69-
}
70-
71-
// upserts {source: target} mappings
72-
// to the fastly edge dictionary
73-
async connectAndUpsert(map) {
74-
const options = {
75-
item_value: map.target
76-
};
77-
const connectString = `/service/${environment.getFastlyServiceId()}/dictionary/${environment.getDochubMap()}/item/${
78-
map.source
79-
}`;
80-
81-
return new Promise((resolve, reject) => {
82-
fastly.request('PUT', connectString, options, function(err, obj) {
83-
if (err) reject(err);
84-
resolve(obj);
85-
});
86-
})
87-
}
122+
})
123+
}
88124
}
89125

90126
module.exports = {
91-
FastlyJobClass: FastlyJobClass
92-
};
127+
FastlyJobClass,
128+
};

0 commit comments

Comments
 (0)