Skip to content

Commit ea1e94c

Browse files
aboseabose
committed
chore: upload extension zip to s3 after validation
1 parent c03fb6f commit ea1e94c

File tree

2 files changed

+59
-29
lines changed

2 files changed

+59
-29
lines changed

‎src/api/publishGithubRelease.js‎

Lines changed: 44 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,13 @@ import {getRepoDetails, getReleaseDetails, createIssue, getOrgDetails} from "../
44
import db from "../db.js";
55
import {downloader} from "../utils/downloader.js";
66
import {ZipUtils} from "../utils/zipUtils.js";
7-
import {valid, lte} from "semver";
7+
import {valid, lte, clean} from "semver";
88
import {
99
FIELD_RELEASE_ID, RELEASE_DETAILS_TABLE, EXTENSION_SIZE_LIMIT_MB, BASE_URL,
10-
EXTENSION_DOWNLOAD_DIR, PROCESSING_TIMEOUT_MS, EXTENSIONS_DETAILS_TABLE, FIELD_EXTENSION_ID
10+
EXTENSION_DOWNLOAD_DIR, PROCESSING_TIMEOUT_MS, EXTENSIONS_DETAILS_TABLE, FIELD_EXTENSION_ID, EXTENSIONS_BUCKET
1111
} from "../constants.js";
12+
import fs from "fs";
13+
import {S3} from "../s3.js";
1214

1315
const RELEASE_STATUS_PROCESSING = "processing";
1416

@@ -156,15 +158,15 @@ async function _validateExtensionPackageJson(githubReleaseTag, packageJSON, repo
156158
const newOwner = `github:${githubReleaseTag.owner}`;
157159
const releaseRef = `${githubReleaseTag.owner}/${githubReleaseTag.repo}/${githubReleaseTag.tag}`;
158160
queryObj[FIELD_EXTENSION_ID] = packageJSON.name;
159-
let registryPKG = await db.getFromIndex(EXTENSIONS_DETAILS_TABLE, queryObj);
160-
if(!registryPKG.isSuccess){
161+
let registryPKGJSON = await db.getFromIndex(EXTENSIONS_DETAILS_TABLE, queryObj);
162+
if(!registryPKGJSON.isSuccess){
161163
// unexpected error
162164
throw new Error("Error getting extensionPKG details from db: " + releaseRef);
163165
}
164-
registryPKG = registryPKG.documents.length === 1 ? registryPKG.documents[0] : null;
166+
registryPKGJSON = registryPKGJSON.documents.length === 1 ? registryPKGJSON.documents[0] : null;
165167
let error = "";
166-
if(registryPKG && registryPKG.owner !== newOwner) {
167-
let errorMsg = `Extension of the same name "${packageJSON.name}" already exists (owned by https://github.com/${registryPKG.owner.split(":")[1]}). Please choose a different extension name.`;
168+
if(registryPKGJSON && registryPKGJSON.owner !== newOwner) {
169+
let errorMsg = `Extension of the same name "${packageJSON.name}" already exists (owned by https://github.com/${registryPKGJSON.owner.split(":")[1]}). Please choose a different extension name.`;
168170
error = error + errorMsg;
169171
issueMessages.push(errorMsg);
170172
throw {status: HTTP_STATUS_CODES.BAD_REQUEST,
@@ -176,17 +178,18 @@ async function _validateExtensionPackageJson(githubReleaseTag, packageJSON, repo
176178
error = error + `\n${errorMsg}`;
177179
issueMessages.push(errorMsg);
178180
}
179-
if(registryPKG) {
180-
for(let versionInfo of registryPKG.versions){
181+
packageJSON.version = clean(packageJSON.version); // ' =v1.2.3 ' -> '1.2.3'
182+
if(registryPKGJSON) {
183+
for(let versionInfo of registryPKGJSON.versions){
181184
if(versionInfo.version === packageJSON.version){
182-
let errorMsg = `Package version "${packageJSON.version}" already published on ${versionInfo.published}. Please update version number to above ${registryPKG.metadata.version}.`;
185+
let errorMsg = `Package version "${packageJSON.version}" already published on ${versionInfo.published}. Please update version number to above ${registryPKGJSON.metadata.version}.`;
183186
error = error + `\n${errorMsg}`;
184187
issueMessages.push(errorMsg);
185188
break;
186189
}
187190
}
188-
if(lte(packageJSON.version, registryPKG.metadata.version)){
189-
let errorMsg = `Package version should be greater than ${registryPKG.metadata.version}, but received "${packageJSON.version}".`;
191+
if(lte(packageJSON.version, registryPKGJSON.metadata.version)){
192+
let errorMsg = `Package version should be greater than ${registryPKGJSON.metadata.version}, but received "${packageJSON.version}".`;
190193
error = error + `\n${errorMsg}`;
191194
issueMessages.push(errorMsg);
192195
}
@@ -202,30 +205,30 @@ async function _validateExtensionPackageJson(githubReleaseTag, packageJSON, repo
202205
ownershipVerifiedByGitHub = [org.blog];
203206
}
204207
// now create the new registry package json
205-
registryPKG = registryPKG || {
208+
registryPKGJSON = registryPKGJSON || {
206209
"versions": [],
207210
"totalDownloads": 0,
208211
"recent": {}
209212
};
210-
registryPKG.metadata= packageJSON;
211-
registryPKG.owner= `github:${githubReleaseTag.owner}`;
212-
registryPKG.gihubStars = repoDetails.stargazers_count;
213-
registryPKG.ownerRepo = `https://github.com/${githubReleaseTag.owner}/${githubReleaseTag.repo}`;
214-
registryPKG.ownershipVerifiedByGitHub = ownershipVerifiedByGitHub;
215-
registryPKG.versions.push({
216-
"version": packageJSON.version,
213+
registryPKGJSON.metadata= packageJSON;
214+
registryPKGJSON.owner= `github:${githubReleaseTag.owner}`;
215+
registryPKGJSON.gihubStars = repoDetails.stargazers_count;
216+
registryPKGJSON.ownerRepo = `https://github.com/${githubReleaseTag.owner}/${githubReleaseTag.repo}`;
217+
registryPKGJSON.ownershipVerifiedByGitHub = ownershipVerifiedByGitHub;
218+
registryPKGJSON.versions.push({
219+
"version": clean(packageJSON.version),
217220
"published": new Date().toISOString(),
218221
"brackets": packageJSON.engines.brackets,
219222
"downloads": 0
220223
});
221224

222-
console.log(registryPKG);
225+
return registryPKGJSON;
223226
}
224227

225228
async function _downloadAndValidateExtensionZip(githubReleaseTag, extensionZipAsset, repoDetails, issueMessages) {
226-
const targetPath = `${EXTENSION_DOWNLOAD_DIR}/${githubReleaseTag.owner}_${githubReleaseTag.repo}_${githubReleaseTag.tag}_${extensionZipAsset.name}`;
227-
await downloader.downloadFile(extensionZipAsset.browser_download_url, targetPath);
228-
let {packageJSON, error} = await ZipUtils.getExtensionPackageJSON(targetPath);
229+
const extensionZipPath = `${EXTENSION_DOWNLOAD_DIR}/${githubReleaseTag.owner}_${githubReleaseTag.repo}_${githubReleaseTag.tag}_${extensionZipAsset.name}`;
230+
await downloader.downloadFile(extensionZipAsset.browser_download_url, extensionZipPath);
231+
let {packageJSON, error} = await ZipUtils.getExtensionPackageJSON(extensionZipPath);
229232
if(error) {
230233
issueMessages.push(error);
231234
throw {status: HTTP_STATUS_CODES.BAD_REQUEST,
@@ -249,8 +252,8 @@ async function _downloadAndValidateExtensionZip(githubReleaseTag, extensionZipAs
249252
updatePublishErrors: true,
250253
error};
251254
}
252-
await _validateExtensionPackageJson(githubReleaseTag, packageJSON, repoDetails, issueMessages);
253-
return targetPath;
255+
const registryPKGJSON = await _validateExtensionPackageJson(githubReleaseTag, packageJSON, repoDetails, issueMessages);
256+
return {extensionZipPath, registryPKGJSON};
254257
}
255258

256259
async function _createGithubIssue(release) {
@@ -340,9 +343,20 @@ export async function publishGithubRelease(request, reply) {
340343
error: `Draft or PreRelease builds cannot be published.`};
341344
}
342345
const extensionZipAsset = _validateGitHubReleaseAssets(newGithubReleaseDetails, issueMessages);
343-
extensionZipPath = await _downloadAndValidateExtensionZip(githubReleaseTag, extensionZipAsset, repoDetails, issueMessages);
346+
const {extensionZipPath, registryPKGJSON}=
347+
await _downloadAndValidateExtensionZip(githubReleaseTag, extensionZipAsset, repoDetails, issueMessages);
344348
// we should also in the future do a virus scan, but will rely on av in users machine for the time being
345349
// https://developers.virustotal.com/reference/files-scan by Google Cloud is available for non-commercial apps.
350+
351+
await S3.uploadFile(EXTENSIONS_BUCKET,
352+
`extensions/${registryPKGJSON.metadata.name}-${registryPKGJSON.metadata.version}.zip`,
353+
extensionZipPath);
354+
355+
// cleanup
356+
setTimeout(()=>{
357+
fs.unlink(extensionZipPath, console.error); // cleanup after we return. (But we don't check the result)
358+
}, 1000);
359+
346360
const response = {
347361
message: "done"
348362
};
@@ -352,6 +366,9 @@ export async function publishGithubRelease(request, reply) {
352366
if(err.updatePublishErrors) {
353367
_updatePublishErrors(githubReleaseTag, issueMessages); // dont await, background task
354368
}
369+
if(extensionZipPath){
370+
fs.unlink(extensionZipPath, console.error); // cleanup after we return. (But we don't check the result)
371+
}
355372
if(err.status){
356373
reply.status(err.status);
357374
return err.error;

‎test/unit/api/publishGithubRelease.spec.js‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import registryJSON from "../data/registry.js";
1111
import Ajv from "ajv";
1212
import {initGitHubClient} from "../../../src/github.js";
1313
import {EXTENSION_SIZE_LIMIT_MB} from "../../../src/constants.js";
14+
import {S3} from '../../../src/s3.js';
1415

1516
export const AJV = new Ajv();
1617

@@ -45,9 +46,21 @@ describe('unit Tests for publishGithubRelease api', function () {
4546
};
4647
});
4748

48-
it('should publishGithubRelease', async function () {
49+
async function _testPublishSuccess() {
50+
let bucket, key, filePathToUpload;
51+
S3.uploadFile = function (_bucket, _key, _filePathToUpload) {
52+
bucket = _bucket; key = _key; filePathToUpload = _filePathToUpload;
53+
};
4954
let helloResponse = await publishGithubRelease(request, reply);
5055
expect(helloResponse).eql({message: 'done'});
56+
expect(bucket).eql("phcode-extensions-test");
57+
expect(key).eql("extensions/angular.moduler-0.0.1.zip");
58+
expect(filePathToUpload.endsWith("downloads/org_repo_gitTag_extension.zip")).to.be.true;
59+
return helloResponse;
60+
}
61+
62+
it('should publishGithubRelease', async function () {
63+
await _testPublishSuccess();
5164
});
5265

5366
it('should validate schemas for sample request/responses', async function () {
@@ -56,7 +69,7 @@ describe('unit Tests for publishGithubRelease api', function () {
5669
expect(requestValidator(request.query)).to.be.true;
5770
// response
5871
const successResponseValidator = AJV.compile(getPublishGithubReleaseSchema().schema.response["200"]);
59-
let response = await publishGithubRelease(request, reply);
72+
let response = await _testPublishSuccess();
6073
expect(successResponseValidator(response)).to.be.true;
6174
});
6275

0 commit comments

Comments
 (0)