From f1c9cca06d2d39037eab2d9186b1936b0cf5a439 Mon Sep 17 00:00:00 2001 From: Anton Lazarev Date: Fri, 10 Nov 2023 16:52:23 -0800 Subject: [PATCH 1/5] remove superfluous directory creation --- lib/util.js | 4 ++++ scripts/ntp-sponsored-images/package.js | 10 ++++------ scripts/packageAdBlock.js | 8 ++++---- scripts/packageBraveAdsResourcesComponent.js | 8 +++----- scripts/packageComponent.js | 5 +++-- scripts/packageIpfsDaemon.js | 9 ++++----- scripts/packageLocalDataFiles.js | 3 +-- scripts/packageNTPBackgroundImagesComponent.js | 7 ++----- scripts/packageNTPSuperReferrerComponent.js | 7 ++----- .../packageNTPSuperReferrerMappingTableComponent.js | 7 ++----- scripts/packagePlaylist.js | 7 ++----- scripts/packageTorClient.js | 9 ++++----- scripts/packageTorPluggableTransports.js | 13 ++++++------- 13 files changed, 41 insertions(+), 56 deletions(-) diff --git a/lib/util.js b/lib/util.js index 31d5b2b6..de753064 100644 --- a/lib/util.js +++ b/lib/util.js @@ -76,8 +76,12 @@ const generateCRXFile = (binary, crxFile, privateKeyFile, publisherProofKey, throw new Error(`Private key file '${privateKeyFile}' is missing, was it uploaded?`) } + const crxOutputDir = path.dirname(crxFile) + mkdirp.sync(crxOutputDir) + const tmp = tmpdir() const tempUserDataDir = fs.mkdtempSync(path.join(tmp, 'crx-package-job-')) + const args = [ `--pack-extension=${path.resolve(inputDir)}`, `--pack-extension-key=${path.resolve(privateKeyFile)}`, diff --git a/scripts/ntp-sponsored-images/package.js b/scripts/ntp-sponsored-images/package.js index bcd18227..09086e7f 100644 --- a/scripts/ntp-sponsored-images/package.js +++ b/scripts/ntp-sponsored-images/package.js @@ -48,15 +48,13 @@ function getManifestPath (regionPlatform) { const generateCRXFile = (binary, endpoint, region, keyDir, platformRegion, componentData, publisherProofKey) => { + // Desktop private key file names do not have the -desktop suffix, but android has -android + const privateKeyFile = path.join(keyDir, `ntp-sponsored-images-${platformRegion.replace('-desktop', '')}.pem`) const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-sponsored-images') + const stagingDir = path.join(rootBuildDir, 'staging', platformRegion) - const crxOutputDir = path.join(rootBuildDir, 'output') - mkdirp.sync(stagingDir) - mkdirp.sync(crxOutputDir) + const crxFile = path.join(rootBuildDir, 'output', `ntp-sponsored-images-${platformRegion}.crx`) util.getNextVersion(endpoint, region, componentData.id).then((version) => { - const crxFile = path.join(crxOutputDir, `ntp-sponsored-images-${platformRegion}.crx`) - // Desktop private key file names do not have the -desktop suffix, but android has -android - const privateKeyFile = path.join(keyDir, `ntp-sponsored-images-${platformRegion.replace('-desktop', '')}.pem`) stageFiles(platformRegion, version, stagingDir) util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, stagingDir) diff --git a/scripts/packageAdBlock.js b/scripts/packageAdBlock.js index d2d3010d..a9c98551 100644 --- a/scripts/packageAdBlock.js +++ b/scripts/packageAdBlock.js @@ -22,17 +22,17 @@ async function stageFiles (version, outputDir) { const postNextVersionWork = (componentSubdir, key, publisherProofKey, binary, localRun, version, contentHash) => { + const crxName = `ad-block-updater-${componentSubdir}` + const privateKeyFile = path.join(key, `${crxName}.pem`) const stagingDir = path.join('build', 'ad-block-updater', componentSubdir) - const crxOutputDir = path.join('build', 'ad-block-updater') - const crxFile = path.join(crxOutputDir, `ad-block-updater-${componentSubdir}.crx`) - const contentHashFile = path.join(crxOutputDir, `ad-block-updater-${componentSubdir}.contentHash`) + const crxFile = path.join('build', 'ad-block-updater', `${crxName}.crx`) + const contentHashFile = path.join('build', 'ad-block-updater', `${crxName}.contentHash`) stageFiles(version, stagingDir).then(() => { // Remove any existing `.contentHash` file for determinism if (fs.existsSync(contentHashFile)) { fs.unlinkSync(contentHashFile) } if (!localRun) { - const privateKeyFile = path.join(key, `ad-block-updater-${componentSubdir}.pem`) util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, stagingDir) } diff --git a/scripts/packageBraveAdsResourcesComponent.js b/scripts/packageBraveAdsResourcesComponent.js index 5f1fa32c..a61cd306 100644 --- a/scripts/packageBraveAdsResourcesComponent.js +++ b/scripts/packageBraveAdsResourcesComponent.js @@ -270,14 +270,12 @@ const getOriginalManifest = (locale) => { const generateCRXFile = (binary, endpoint, region, keyDir, publisherProofKey, componentData) => { const locale = componentData.locale + const privateKeyFile = path.join(keyDir, `user-model-installer-${locale}.pem`) const rootBuildDir = path.join(path.resolve(), 'build', 'user-model-installer') + const stagingDir = path.join(rootBuildDir, 'staging', locale) - const crxOutputDir = path.join(rootBuildDir, 'output') - mkdirp.sync(stagingDir) - mkdirp.sync(crxOutputDir) + const crxFile = path.join(rootBuildDir, 'output', `user-model-installer-${locale}.crx`) util.getNextVersion(endpoint, region, componentData.id).then((version) => { - const crxFile = path.join(crxOutputDir, `user-model-installer-${locale}.crx`) - const privateKeyFile = path.join(keyDir, `user-model-installer-${locale}.pem`) stageFiles(locale, version, stagingDir) util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, stagingDir) diff --git a/scripts/packageComponent.js b/scripts/packageComponent.js index ce66feea..479b64b6 100644 --- a/scripts/packageComponent.js +++ b/scripts/packageComponent.js @@ -44,12 +44,13 @@ const getOriginalManifest = (componentType) => { const postNextVersionWork = (componentType, key, publisherProofKey, binary, localRun, version) => { - const stagingDir = path.join('build', componentType) - const crxFile = path.join(stagingDir, `${componentType}.crx`) let privateKeyFile = '' if (!localRun) { privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `${componentType}.pem`) } + + const stagingDir = path.join('build', componentType) + const crxFile = path.join(stagingDir, `${componentType}.crx`) stageFiles(componentType, version, stagingDir) if (!localRun) { util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, diff --git a/scripts/packageIpfsDaemon.js b/scripts/packageIpfsDaemon.js index 52971dbb..7c8ef7c7 100644 --- a/scripts/packageIpfsDaemon.js +++ b/scripts/packageIpfsDaemon.js @@ -25,16 +25,15 @@ const getOriginalManifest = (platform) => { const packageIpfsDaemon = (binary, endpoint, region, os, arch, key, publisherProofKey) => { const platform = `${os}-${arch}` + const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `ipfs-daemon-updater-${platform}.pem`) const originalManifest = getOriginalManifest(platform) const parsedManifest = util.parseManifest(originalManifest) const id = util.getIDFromBase64PublicKey(parsedManifest.key) + const ipfsDaemon = getIpfsDaemonPath(os, arch) + const stagingDir = path.join('build', 'ipfs-daemon-updater', platform) + const crxFile = path.join('build', 'ipfs-daemon-updater', `ipfs-daemon-updater-${platform}.crx`) util.getNextVersion(endpoint, region, id).then((version) => { - const stagingDir = path.join('build', 'ipfs-daemon-updater', platform) - const ipfsDaemon = getIpfsDaemonPath(os, arch) - const crxOutputDir = path.join('build', 'ipfs-daemon-updater') - const crxFile = path.join(crxOutputDir, `ipfs-daemon-updater-${platform}.crx`) - const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `ipfs-daemon-updater-${platform}.pem`) stageFiles(platform, ipfsDaemon, version, stagingDir) util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, stagingDir) diff --git a/scripts/packageLocalDataFiles.js b/scripts/packageLocalDataFiles.js index 77dd7c9e..cdd547f2 100644 --- a/scripts/packageLocalDataFiles.js +++ b/scripts/packageLocalDataFiles.js @@ -45,8 +45,7 @@ const postNextVersionWork = (key, publisherProofKey, binary, localRun, version) const componentType = 'local-data-files-updater' const datFileName = 'default' const stagingDir = path.join('build', componentType, datFileName) - const crxOutputDir = path.join('build', componentType) - const crxFile = path.join(crxOutputDir, `${componentType}-${datFileName}.crx`) + const crxFile = path.join('build', componentType, `${componentType}-${datFileName}.crx`) let privateKeyFile = '' if (!localRun) { privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `${componentType}-${datFileName}.pem`) diff --git a/scripts/packageNTPBackgroundImagesComponent.js b/scripts/packageNTPBackgroundImagesComponent.js index 8839119b..26c0a505 100644 --- a/scripts/packageNTPBackgroundImagesComponent.js +++ b/scripts/packageNTPBackgroundImagesComponent.js @@ -4,7 +4,6 @@ import commander from 'commander' import fs from 'fs-extra' -import { mkdirp } from 'mkdirp' import path from 'path' import util from '../lib/util.js' import ntpUtil from '../lib/ntpUtil.js' @@ -33,12 +32,10 @@ const generateManifestFile = (publicKey) => { const generateCRXFile = (binary, endpoint, region, componentID, privateKeyFile, publisherProofKey) => { const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-background-images') + const stagingDir = path.join(rootBuildDir, 'staging') - const crxOutputDir = path.join(rootBuildDir, 'output') - mkdirp.sync(stagingDir) - mkdirp.sync(crxOutputDir) + const crxFile = path.join(rootBuildDir, 'output', 'ntp-background-images.crx') util.getNextVersion(endpoint, region, componentID).then((version) => { - const crxFile = path.join(crxOutputDir, 'ntp-background-images.crx') stageFiles(version, stagingDir) util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, stagingDir) diff --git a/scripts/packageNTPSuperReferrerComponent.js b/scripts/packageNTPSuperReferrerComponent.js index 7d4cb7df..029d0fe0 100644 --- a/scripts/packageNTPSuperReferrerComponent.js +++ b/scripts/packageNTPSuperReferrerComponent.js @@ -4,7 +4,6 @@ import commander from 'commander' import fs from 'fs-extra' -import { mkdirp } from 'mkdirp' import path from 'path' import util from '../lib/util.js' import ntpUtil from '../lib/ntpUtil.js' @@ -36,12 +35,10 @@ const getOriginalManifest = (superReferrerName) => { const generateCRXFile = (binary, endpoint, region, superReferrerName, componentID, privateKeyFile, publisherProofKey) => { const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer') + const stagingDir = path.join(rootBuildDir, 'staging', superReferrerName) - const crxOutputDir = path.join(rootBuildDir, 'output') - mkdirp.sync(stagingDir) - mkdirp.sync(crxOutputDir) + const crxFile = path.join(rootBuildDir, 'output', `ntp-super-referrer-${superReferrerName}.crx`) util.getNextVersion(endpoint, region, componentID).then((version) => { - const crxFile = path.join(crxOutputDir, `ntp-super-referrer-${superReferrerName}.crx`) stageFiles(superReferrerName, version, stagingDir) util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, stagingDir) diff --git a/scripts/packageNTPSuperReferrerMappingTableComponent.js b/scripts/packageNTPSuperReferrerMappingTableComponent.js index 499db25c..eaac6d0d 100644 --- a/scripts/packageNTPSuperReferrerMappingTableComponent.js +++ b/scripts/packageNTPSuperReferrerMappingTableComponent.js @@ -4,7 +4,6 @@ import commander from 'commander' import fs from 'fs-extra' -import { mkdirp } from 'mkdirp' import path from 'path' import util from '../lib/util.js' import ntpUtil from '../lib/ntpUtil.js' @@ -36,12 +35,10 @@ const getOriginalManifest = () => { const generateCRXFile = (binary, endpoint, region, componentID, privateKeyFile, publisherProofKey) => { const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer', 'mapping-table') + const stagingDir = path.join(rootBuildDir, 'staging') - const crxOutputDir = path.join(rootBuildDir, 'output') - mkdirp.sync(stagingDir) - mkdirp.sync(crxOutputDir) + const crxFile = path.join(rootBuildDir, 'output', 'ntp-super-referrer-mapping-table.crx') util.getNextVersion(endpoint, region, componentID).then((version) => { - const crxFile = path.join(crxOutputDir, 'ntp-super-referrer-mapping-table.crx') stageFiles(version, stagingDir) util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, stagingDir) diff --git a/scripts/packagePlaylist.js b/scripts/packagePlaylist.js index 1654184c..0fa27b99 100644 --- a/scripts/packagePlaylist.js +++ b/scripts/packagePlaylist.js @@ -4,7 +4,6 @@ import commander from 'commander' import fs from 'fs-extra' -import { mkdirp } from 'mkdirp' import path from 'path' import util from '../lib/util.js' import ntpUtil from '../lib/ntpUtil.js' @@ -21,12 +20,10 @@ const stageFiles = util.stageDir.bind( const generateCRXFile = (binary, endpoint, region, componentID, privateKeyFile, publisherProofKey) => { const rootBuildDir = path.join(path.resolve(), 'build', 'playlist') + const stagingDir = path.join(rootBuildDir, 'staging') - const crxOutputDir = path.join(rootBuildDir, 'output') - mkdirp.sync(stagingDir) - mkdirp.sync(crxOutputDir) + const crxFile = path.join(rootBuildDir, 'output', 'playlist.crx') util.getNextVersion(endpoint, region, componentID).then((version) => { - const crxFile = path.join(crxOutputDir, 'playlist.crx') stageFiles(version, stagingDir) util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, stagingDir) diff --git a/scripts/packageTorClient.js b/scripts/packageTorClient.js index 99bf279f..74f3d1e5 100644 --- a/scripts/packageTorClient.js +++ b/scripts/packageTorClient.js @@ -69,16 +69,15 @@ const getOriginalManifest = (platform) => { const packageTorClient = (binary, endpoint, region, platform, key, publisherProofKey) => { + const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `tor-client-updater-${platform}.pem`) const originalManifest = getOriginalManifest(platform) const parsedManifest = util.parseManifest(originalManifest) const id = util.getIDFromBase64PublicKey(parsedManifest.key) + const torClient = downloadTorClient(platform) + const stagingDir = path.join('build', 'tor-client-updater', platform) + const crxFile = path.join('build', 'tor-client-updater', `tor-client-updater-${platform}.crx`) util.getNextVersion(endpoint, region, id).then((version) => { - const stagingDir = path.join('build', 'tor-client-updater', platform) - const torClient = downloadTorClient(platform) - const crxOutputDir = path.join('build', 'tor-client-updater') - const crxFile = path.join(crxOutputDir, `tor-client-updater-${platform}.crx`) - const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `tor-client-updater-${platform}.pem`) stageFiles(platform, torClient, version, stagingDir) util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, stagingDir) diff --git a/scripts/packageTorPluggableTransports.js b/scripts/packageTorPluggableTransports.js index 11cef25c..08b9de85 100644 --- a/scripts/packageTorPluggableTransports.js +++ b/scripts/packageTorPluggableTransports.js @@ -46,18 +46,17 @@ const getOriginalManifest = (platform) => { } const packageTorPluggableTransports = (binary, endpoint, region, platform, key, publisherProofKey) => { + const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `${TOR_PLUGGABLE_TRANSPORTS_UPDATER}-${platform}.pem`) const originalManifest = getOriginalManifest(platform) const parsedManifest = util.parseManifest(originalManifest) const id = util.getIDFromBase64PublicKey(parsedManifest.key) - util.getNextVersion(endpoint, region, id).then((version) => { - const snowflake = downloadTorPluggableTransport(platform, 'snowflake') - const obfs4 = downloadTorPluggableTransport(platform, 'obfs4') + const snowflake = downloadTorPluggableTransport(platform, 'snowflake') + const obfs4 = downloadTorPluggableTransport(platform, 'obfs4') - const stagingDir = path.join('build', TOR_PLUGGABLE_TRANSPORTS_UPDATER, platform) - const crxOutputDir = path.join('build', TOR_PLUGGABLE_TRANSPORTS_UPDATER) - const crxFile = path.join(crxOutputDir, `${TOR_PLUGGABLE_TRANSPORTS_UPDATER}-${platform}.crx`) - const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `${TOR_PLUGGABLE_TRANSPORTS_UPDATER}-${platform}.pem`) + const stagingDir = path.join('build', TOR_PLUGGABLE_TRANSPORTS_UPDATER, platform) + const crxFile = path.join('build', TOR_PLUGGABLE_TRANSPORTS_UPDATER, `${TOR_PLUGGABLE_TRANSPORTS_UPDATER}-${platform}.crx`) + util.getNextVersion(endpoint, region, id).then((version) => { stageFiles(platform, snowflake, obfs4, version, stagingDir) util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, stagingDir) console.log(`Generated ${crxFile} with version number ${version}`) From 83b48fd8804c004d06173ad6219b65eca2355761 Mon Sep 17 00:00:00 2001 From: Anton Lazarev Date: Fri, 10 Nov 2023 19:06:14 -0800 Subject: [PATCH 2/5] absorb more common packaging logic into util.prepareNextVersionCRX --- lib/util.js | 40 ++++++++- scripts/ntp-sponsored-images/package.js | 24 ++++-- scripts/packageAdBlock.js | 85 ++++++++----------- scripts/packageBraveAdsResourcesComponent.js | 26 +++--- scripts/packageComponent.js | 50 +++++------ scripts/packageIpfsDaemon.js | 34 +++++--- scripts/packageLocalDataFiles.js | 50 +++++------ .../packageNTPBackgroundImagesComponent.js | 24 ++++-- scripts/packageNTPSuperReferrerComponent.js | 24 ++++-- ...geNTPSuperReferrerMappingTableComponent.js | 26 +++--- scripts/packagePlaylist.js | 24 ++++-- scripts/packageTorClient.js | 30 ++++--- scripts/packageTorPluggableTransports.js | 27 +++--- 13 files changed, 262 insertions(+), 202 deletions(-) diff --git a/lib/util.js b/lib/util.js index de753064..adcd147c 100644 --- a/lib/util.js +++ b/lib/util.js @@ -64,6 +64,41 @@ const fetchTextFromURL = (listURL) => { return p } +const prepareNextVersionCRX = async (binary, publisherProofKey, endpoint, region, componentId, stageFilesFn, stagingDir, outputCrx, privateKeyFile, localRun, contentHashFn, contentHashFile) => { + let contentHash + if (contentHashFn !== undefined) { + contentHash = contentHashFn() + } + + let version + if (!localRun) { + version = '1.0.0' + } else { + version = await getNextVersion(endpoint, region, componentId, contentHash) + if (version === undefined) { + console.log(`content for ${componentId} was not updated, skipping!`) + return + } + } + + // Remove any existing `.contentHash` file for determinism + if (contentHashFile !== undefined && fs.existsSync(contentHashFile)) { + fs.unlinkSync(contentHashFile) + } + + stageFilesFn(version, stagingDir) + + if (!localRun) { + generateCRXFile(binary, outputCrx, privateKeyFile, publisherProofKey, stagingDir) + } + + if (contentHash !== undefined && contentHashFile !== undefined) { + fs.writeFileSync(contentHashFile, contentHash) + } + + console.log(`Generated ${outputCrx} with version number ${version}`) +} + const generateCRXFile = (binary, crxFile, privateKeyFile, publisherProofKey, inputDir) => { if (!binary) { @@ -617,10 +652,8 @@ export default { fetchTextFromURL, createTableIfNotExists, fetchPreviousVersions, - generateCRXFile, generatePuffPatches, generateSHA256HashOfFile, - getNextVersion, getIDFromBase64PublicKey, installErrorHandlers, parseManifest, @@ -630,5 +663,6 @@ export default { escapeStringForJSON, copyManifestWithVersion, stageDir, - stageFiles + stageFiles, + prepareNextVersionCRX } diff --git a/scripts/ntp-sponsored-images/package.js b/scripts/ntp-sponsored-images/package.js index 09086e7f..cdb71201 100644 --- a/scripts/ntp-sponsored-images/package.js +++ b/scripts/ntp-sponsored-images/package.js @@ -46,7 +46,7 @@ function getManifestPath (regionPlatform) { return path.join(getManifestsDir(), `${regionPlatform}-manifest.json`) } -const generateCRXFile = (binary, endpoint, region, keyDir, platformRegion, +const generateCRXFile = async (binary, endpoint, region, keyDir, platformRegion, componentData, publisherProofKey) => { // Desktop private key file names do not have the -desktop suffix, but android has -android const privateKeyFile = path.join(keyDir, `ntp-sponsored-images-${platformRegion.replace('-desktop', '')}.pem`) @@ -54,12 +54,18 @@ const generateCRXFile = (binary, endpoint, region, keyDir, platformRegion, const stagingDir = path.join(rootBuildDir, 'staging', platformRegion) const crxFile = path.join(rootBuildDir, 'output', `ntp-sponsored-images-${platformRegion}.crx`) - util.getNextVersion(endpoint, region, componentData.id).then((version) => { - stageFiles(platformRegion, version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) + + await util.prepareNextVersionCRX( + binary, + publisherProofKey, + endpoint, + region, + componentData.id, + stageFiles.bind(undefined, platformRegion), + stagingDir, + crxFile, + privateKeyFile, + false) } util.installErrorHandlers() @@ -80,11 +86,11 @@ if (fs.existsSync(commander.keysDirectory)) { const targetComponents = params.getTargetComponents(commander.targetRegions, commander.excludedTargetRegions) -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { +util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { for (const platformRegion of Object.keys(targetComponents)) { const componentData = targetComponents[platformRegion] generateManifestFile(platformRegion, componentData) - generateCRXFile(commander.binary, commander.endpoint, commander.region, + await generateCRXFile(commander.binary, commander.endpoint, commander.region, keyDir, platformRegion, componentData, commander.publisherProofKey) } diff --git a/scripts/packageAdBlock.js b/scripts/packageAdBlock.js index a9c98551..ac5e66e2 100644 --- a/scripts/packageAdBlock.js +++ b/scripts/packageAdBlock.js @@ -12,7 +12,7 @@ import path from 'path' import util from '../lib/util.js' import { getListCatalog, regionalCatalogComponentId, resourcesComponentId } from '../lib/adBlockRustUtils.js' -async function stageFiles (version, outputDir) { +function stageFiles (version, outputDir) { // ad-block components are already written in the output directory // so we don't need to stage anything const originalManifest = path.join(outputDir, 'manifest.json') @@ -20,68 +20,53 @@ async function stageFiles (version, outputDir) { util.copyManifestWithVersion(originalManifest, outputDir, version) } -const postNextVersionWork = (componentSubdir, key, publisherProofKey, - binary, localRun, version, contentHash) => { - const crxName = `ad-block-updater-${componentSubdir}` - const privateKeyFile = path.join(key, `${crxName}.pem`) - const stagingDir = path.join('build', 'ad-block-updater', componentSubdir) - const crxFile = path.join('build', 'ad-block-updater', `${crxName}.crx`) - const contentHashFile = path.join('build', 'ad-block-updater', `${crxName}.contentHash`) - stageFiles(version, stagingDir).then(() => { - // Remove any existing `.contentHash` file for determinism - if (fs.existsSync(contentHashFile)) { - fs.unlinkSync(contentHashFile) - } - if (!localRun) { - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - } - if (contentHash !== undefined) { - fs.writeFileSync(contentHashFile, contentHash) - } - console.log(`Generated ${crxFile} with version number ${version}`) - }) -} - const getOriginalManifest = (componentSubdir) => { const manifestsDir = path.join('build', 'ad-block-updater') return path.join(manifestsDir, componentSubdir, 'manifest.json') } -const processComponent = (binary, endpoint, region, keyDir, +const processComponent = async (binary, endpoint, region, keyDir, publisherProofKey, localRun, componentSubdir) => { const originalManifest = getOriginalManifest(componentSubdir) const parsedManifest = util.parseManifest(originalManifest) const id = util.getIDFromBase64PublicKey(parsedManifest.key) - let fileToHash - if (componentSubdir === regionalCatalogComponentId) { - fileToHash = 'regional_catalog.json' - } else if (componentSubdir === resourcesComponentId) { - fileToHash = 'resources.json' - } else { - fileToHash = 'list.txt' - } + const crxName = `ad-block-updater-${componentSubdir}` + const privateKeyFile = localRun ? undefined : path.join(keyDir, `${crxName}.pem`) + const stagingDir = path.join('build', 'ad-block-updater', componentSubdir) + const crxFile = path.join('build', 'ad-block-updater', `${crxName}.crx`) + const contentHashFile = path.join('build', 'ad-block-updater', `${crxName}.contentHash`) - let contentHash - if (fileToHash !== undefined) { - const contentFile = path.join('build', 'ad-block-updater', componentSubdir, fileToHash) - contentHash = util.generateSHA256HashOfFile(contentFile) + const contentHashFn = () => { + let fileToHash + if (componentSubdir === regionalCatalogComponentId) { + fileToHash = 'regional_catalog.json' + } else if (componentSubdir === resourcesComponentId) { + fileToHash = 'resources.json' + } else { + fileToHash = 'list.txt' + } + if (fileToHash !== undefined) { + const contentFile = path.join('build', 'ad-block-updater', componentSubdir, fileToHash) + return util.generateSHA256HashOfFile(contentFile) + } else { + return undefined + } } - if (!localRun) { - util.getNextVersion(endpoint, region, id, contentHash).then((version) => { - if (version !== undefined) { - postNextVersionWork(componentSubdir, keyDir, publisherProofKey, - binary, localRun, version, contentHash) - } else { - console.log('content for ' + id + ' was not updated, skipping!') - } - }) - } else { - postNextVersionWork(componentSubdir, undefined, publisherProofKey, - binary, localRun, '1.0.0', contentHash) - } + await util.prepareNextVersionCRX( + binary, + publisherProofKey, + endpoint, + region, + id, + stageFiles, + stagingDir, + crxFile, + privateKeyFile, + localRun, + contentHashFn, + contentHashFile) } const getComponentList = async () => { diff --git a/scripts/packageBraveAdsResourcesComponent.js b/scripts/packageBraveAdsResourcesComponent.js index a61cd306..026087c2 100644 --- a/scripts/packageBraveAdsResourcesComponent.js +++ b/scripts/packageBraveAdsResourcesComponent.js @@ -267,7 +267,7 @@ const getOriginalManifest = (locale) => { return path.join(getManifestsDir(), `${locale}-manifest.json`) } -const generateCRXFile = (binary, endpoint, region, keyDir, publisherProofKey, +const generateCRXFile = async (binary, endpoint, region, keyDir, publisherProofKey, componentData) => { const locale = componentData.locale const privateKeyFile = path.join(keyDir, `user-model-installer-${locale}.pem`) @@ -275,12 +275,18 @@ const generateCRXFile = (binary, endpoint, region, keyDir, publisherProofKey, const stagingDir = path.join(rootBuildDir, 'staging', locale) const crxFile = path.join(rootBuildDir, 'output', `user-model-installer-${locale}.crx`) - util.getNextVersion(endpoint, region, componentData.id).then((version) => { - stageFiles(locale, version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) + + util.prepareNextVersionCRX( + binary, + publisherProofKey, + endpoint, + region, + componentData.id, + stageFiles.bind(undefined, locale), + stagingDir, + crxFile, + privateKeyFile, + false) } util.installErrorHandlers() @@ -297,10 +303,10 @@ if (fs.existsSync(commander.keysDirectory)) { throw new Error('Missing or invalid private key directory') } -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { +util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { generateManifestFiles() - getComponentDataList().forEach( + await Promise.all(getComponentDataList().map( generateCRXFile.bind(null, commander.binary, commander.endpoint, commander.region, keyDir, - commander.publisherProofKey)) + commander.publisherProofKey))) }) diff --git a/scripts/packageComponent.js b/scripts/packageComponent.js index 479b64b6..655c9760 100644 --- a/scripts/packageComponent.js +++ b/scripts/packageComponent.js @@ -42,45 +42,37 @@ const getOriginalManifest = (componentType) => { return path.join(getPackageDirByComponentType(componentType), 'manifest.json') } -const postNextVersionWork = (componentType, key, publisherProofKey, - binary, localRun, version) => { - let privateKeyFile = '' - if (!localRun) { - privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `${componentType}.pem`) - } - - const stagingDir = path.join('build', componentType) - const crxFile = path.join(stagingDir, `${componentType}.crx`) - stageFiles(componentType, version, stagingDir) - if (!localRun) { - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - } - console.log(`Generated ${crxFile} with version number ${version}`) -} - -const processDATFile = (binary, endpoint, region, componentType, key, +const generateCRXFile = async (binary, endpoint, region, componentType, key, publisherProofKey, localRun) => { const originalManifest = getOriginalManifest(componentType) const parsedManifest = util.parseManifest(originalManifest) const id = util.getIDFromBase64PublicKey(parsedManifest.key) + let privateKeyFile = '' if (!localRun) { - util.getNextVersion(endpoint, region, id).then((version) => { - postNextVersionWork(componentType, key, publisherProofKey, - binary, localRun, version) - }) - } else { - postNextVersionWork(componentType, key, publisherProofKey, - binary, localRun, '1.0.0') + privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `${componentType}.pem`) } + const stagingDir = path.join('build', componentType) + const crxFile = path.join(stagingDir, `${componentType}.crx`) + + await util.prepareNextVersionCRX( + binary, + publisherProofKey, + endpoint, + region, + id, + stageFiles.bind(undefined, componentType), + stagingDir, + crxFile, + privateKeyFile, + localRun) } -const processJob = (commander, keyParam) => { +const processJob = async (commander, keyParam) => { if (!validComponentTypes.includes(commander.type)) { throw new Error('Unrecognized component extension type: ' + commander.type) } - processDATFile(commander.binary, commander.endpoint, + await generateCRXFile(commander.binary, commander.endpoint, commander.region, commander.type, keyParam, commander.publisherProofKey, commander.localRun) @@ -109,8 +101,8 @@ if (!commander.localRun) { } if (!commander.localRun) { - util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - processJob(commander, keyParam) + util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { + await processJob(commander, keyParam) }) } else { processJob(commander, keyParam) diff --git a/scripts/packageIpfsDaemon.js b/scripts/packageIpfsDaemon.js index 7c8ef7c7..14d41631 100644 --- a/scripts/packageIpfsDaemon.js +++ b/scripts/packageIpfsDaemon.js @@ -22,7 +22,7 @@ const getOriginalManifest = (platform) => { return path.join('manifests', 'ipfs-daemon-updater', `ipfs-daemon-updater-${platform}-manifest.json`) } -const packageIpfsDaemon = (binary, endpoint, region, os, arch, key, +const packageIpfsDaemon = async (binary, endpoint, region, os, arch, key, publisherProofKey) => { const platform = `${os}-${arch}` const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `ipfs-daemon-updater-${platform}.pem`) @@ -33,12 +33,18 @@ const packageIpfsDaemon = (binary, endpoint, region, os, arch, key, const stagingDir = path.join('build', 'ipfs-daemon-updater', platform) const crxFile = path.join('build', 'ipfs-daemon-updater', `ipfs-daemon-updater-${platform}.crx`) - util.getNextVersion(endpoint, region, id).then((version) => { - stageFiles(platform, ipfsDaemon, version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) + + await util.prepareNextVersionCRX( + binary, + publisherProofKey, + endpoint, + region, + id, + stageFiles.bind(undefined, platform, ipfsDaemon), + stagingDir, + crxFile, + privateKeyFile, + false) } const stageFiles = (platform, ipfsDaemon, version, outputDir) => { @@ -67,17 +73,17 @@ if (fs.existsSync(commander.keyFile)) { throw new Error('Missing or invalid private key file/directory') } -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, +util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { + await packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, 'darwin', 'amd64', keyParam, commander.publisherProofKey) - packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, + await packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, 'darwin', 'arm64', keyParam, commander.publisherProofKey) - packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, + await packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, 'linux', 'amd64', keyParam, commander.publisherProofKey) - packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, + await packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, 'linux', 'arm64', keyParam, commander.publisherProofKey) - packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, + await packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, 'win32', 'amd64', keyParam, commander.publisherProofKey) - packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, + await packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, 'win32', 'arm64', keyParam, commander.publisherProofKey) }) diff --git a/scripts/packageLocalDataFiles.js b/scripts/packageLocalDataFiles.js index cdd547f2..4b641d80 100644 --- a/scripts/packageLocalDataFiles.js +++ b/scripts/packageLocalDataFiles.js @@ -41,41 +41,35 @@ const stageFiles = (version, outputDir) => { util.stageFiles(files, version, outputDir) } -const postNextVersionWork = (key, publisherProofKey, binary, localRun, version) => { +const generateCRXFile = async (binary, endpoint, region, key, publisherProofKey, localRun) => { + const originalManifest = getOriginalManifest() + const parsedManifest = util.parseManifest(originalManifest) + const id = util.getIDFromBase64PublicKey(parsedManifest.key) + const componentType = 'local-data-files-updater' const datFileName = 'default' - const stagingDir = path.join('build', componentType, datFileName) - const crxFile = path.join('build', componentType, `${componentType}-${datFileName}.crx`) let privateKeyFile = '' if (!localRun) { privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `${componentType}-${datFileName}.pem`) } - stageFiles(version, stagingDir) - if (!localRun) { - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - } - console.log(`Generated ${crxFile} with version number ${version}`) -} - -const processDATFile = (binary, endpoint, region, key, publisherProofKey, localRun) => { - const originalManifest = getOriginalManifest() - const parsedManifest = util.parseManifest(originalManifest) - const id = util.getIDFromBase64PublicKey(parsedManifest.key) + const stagingDir = path.join('build', componentType, datFileName) + const crxFile = path.join('build', componentType, `${componentType}-${datFileName}.crx`) - if (!localRun) { - util.getNextVersion(endpoint, region, id).then((version) => { - postNextVersionWork(key, publisherProofKey, - binary, localRun, version) - }) - } else { - postNextVersionWork(key, publisherProofKey, - binary, localRun, '1.0.0') - } + await util.prepareNextVersionCRX( + binary, + publisherProofKey, + endpoint, + region, + id, + stageFiles, + stagingDir, + crxFile, + privateKeyFile, + localRun) } -const processJob = (commander, keyParam) => { - processDATFile(commander.binary, commander.endpoint, commander.region, +const processJob = async (commander, keyParam) => { + await generateCRXFile(commander.binary, commander.endpoint, commander.region, keyParam, commander.publisherProofKey, commander.localRun) } @@ -101,8 +95,8 @@ if (!commander.localRun) { } if (!commander.localRun) { - util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - processJob(commander, keyParam) + util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { + await processJob(commander, keyParam) }) } else { processJob(commander, keyParam) diff --git a/scripts/packageNTPBackgroundImagesComponent.js b/scripts/packageNTPBackgroundImagesComponent.js index 26c0a505..12f11f5d 100644 --- a/scripts/packageNTPBackgroundImagesComponent.js +++ b/scripts/packageNTPBackgroundImagesComponent.js @@ -29,18 +29,24 @@ const generateManifestFile = (publicKey) => { fs.writeFileSync(manifestFile, JSON.stringify(manifestContent)) } -const generateCRXFile = (binary, endpoint, region, componentID, privateKeyFile, +const generateCRXFile = async (binary, endpoint, region, componentID, privateKeyFile, publisherProofKey) => { const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-background-images') const stagingDir = path.join(rootBuildDir, 'staging') const crxFile = path.join(rootBuildDir, 'output', 'ntp-background-images.crx') - util.getNextVersion(endpoint, region, componentID).then((version) => { - stageFiles(version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) + + await util.prepareNextVersionCRX( + binary, + publisherProofKey, + endpoint, + region, + componentID, + stageFiles, + stagingDir, + crxFile, + privateKeyFile, + false) } util.installErrorHandlers() @@ -57,9 +63,9 @@ if (fs.existsSync(commander.key)) { throw new Error('Missing or invalid private key') } -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { +util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { const [publicKey, componentID] = ntpUtil.generatePublicKeyAndID(privateKeyFile) generateManifestFile(publicKey) - generateCRXFile(commander.binary, commander.endpoint, commander.region, + await generateCRXFile(commander.binary, commander.endpoint, commander.region, componentID, privateKeyFile, commander.publisherProofKey) }) diff --git a/scripts/packageNTPSuperReferrerComponent.js b/scripts/packageNTPSuperReferrerComponent.js index 029d0fe0..3594786d 100644 --- a/scripts/packageNTPSuperReferrerComponent.js +++ b/scripts/packageNTPSuperReferrerComponent.js @@ -32,18 +32,24 @@ const getOriginalManifest = (superReferrerName) => { return path.join(path.resolve(), 'build', 'ntp-super-referrer', `${superReferrerName}-manifest.json`) } -const generateCRXFile = (binary, endpoint, region, superReferrerName, +const generateCRXFile = async (binary, endpoint, region, superReferrerName, componentID, privateKeyFile, publisherProofKey) => { const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer') const stagingDir = path.join(rootBuildDir, 'staging', superReferrerName) const crxFile = path.join(rootBuildDir, 'output', `ntp-super-referrer-${superReferrerName}.crx`) - util.getNextVersion(endpoint, region, componentID).then((version) => { - stageFiles(superReferrerName, version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) + + await util.prepareNextVersionCRX( + binary, + publisherProofKey, + endpoint, + region, + componentID, + stageFiles.bind(undefined, superReferrerName), + stagingDir, + crxFile, + privateKeyFile, + false) } util.installErrorHandlers() @@ -61,10 +67,10 @@ if (fs.existsSync(commander.key)) { throw new Error('Missing or invalid private key') } -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { +util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { const [publicKey, componentID] = ntpUtil.generatePublicKeyAndID(privateKeyFile) generateManifestFile(commander.superReferrerName, publicKey) - generateCRXFile(commander.binary, commander.endpoint, commander.region, + await generateCRXFile(commander.binary, commander.endpoint, commander.region, commander.superReferrerName, componentID, privateKeyFile, commander.publisherProofKey) }) diff --git a/scripts/packageNTPSuperReferrerMappingTableComponent.js b/scripts/packageNTPSuperReferrerMappingTableComponent.js index eaac6d0d..3dc47c0e 100644 --- a/scripts/packageNTPSuperReferrerMappingTableComponent.js +++ b/scripts/packageNTPSuperReferrerMappingTableComponent.js @@ -16,7 +16,7 @@ const stageFiles = (version, outputDir) => { util.stageFiles(files, version, outputDir) } -const generateManifestFile = (publicKey) => { +const generateManifestFile = async (publicKey) => { const manifestFile = getOriginalManifest() const manifestContent = { description: 'Brave NTP Super Referrer mapping table component', @@ -32,18 +32,24 @@ const getOriginalManifest = () => { return path.join(path.resolve(), 'build', 'ntp-super-referrer', 'mapping-table-manifest.json') } -const generateCRXFile = (binary, endpoint, region, componentID, privateKeyFile, +const generateCRXFile = async (binary, endpoint, region, componentID, privateKeyFile, publisherProofKey) => { const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer', 'mapping-table') const stagingDir = path.join(rootBuildDir, 'staging') const crxFile = path.join(rootBuildDir, 'output', 'ntp-super-referrer-mapping-table.crx') - util.getNextVersion(endpoint, region, componentID).then((version) => { - stageFiles(version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) + + await util.prepareNextVersionCRX( + binary, + publisherProofKey, + endpoint, + region, + componentID, + stageFiles, + stagingDir, + crxFile, + privateKeyFile, + false) } util.installErrorHandlers() @@ -60,9 +66,9 @@ if (fs.existsSync(commander.key)) { throw new Error('Missing or invalid private key') } -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { +util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { const [publicKey, componentID] = ntpUtil.generatePublicKeyAndID(privateKeyFile) generateManifestFile(publicKey) - generateCRXFile(commander.binary, commander.endpoint, commander.region, + await generateCRXFile(commander.binary, commander.endpoint, commander.region, componentID, privateKeyFile, commander.publisherProofKey) }) diff --git a/scripts/packagePlaylist.js b/scripts/packagePlaylist.js index 0fa27b99..d77d578d 100644 --- a/scripts/packagePlaylist.js +++ b/scripts/packagePlaylist.js @@ -17,18 +17,24 @@ const stageFiles = util.stageDir.bind( path.join(path.resolve(), 'node_modules', 'playlist-component'), getOriginalManifest()) -const generateCRXFile = (binary, endpoint, region, componentID, privateKeyFile, +const generateCRXFile = async (binary, endpoint, region, componentID, privateKeyFile, publisherProofKey) => { const rootBuildDir = path.join(path.resolve(), 'build', 'playlist') const stagingDir = path.join(rootBuildDir, 'staging') const crxFile = path.join(rootBuildDir, 'output', 'playlist.crx') - util.getNextVersion(endpoint, region, componentID).then((version) => { - stageFiles(version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) + + await util.prepareNextVersionCRX( + binary, + publisherProofKey, + endpoint, + region, + componentID, + stageFiles, + stagingDir, + crxFile, + privateKeyFile, + false) } util.installErrorHandlers() @@ -45,8 +51,8 @@ if (fs.existsSync(commander.key)) { throw new Error('Missing or invalid private key') } -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { +util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { const [, componentID] = ntpUtil.generatePublicKeyAndID(privateKeyFile) - generateCRXFile(commander.binary, commander.endpoint, commander.region, + await generateCRXFile(commander.binary, commander.endpoint, commander.region, componentID, privateKeyFile, commander.publisherProofKey) }) diff --git a/scripts/packageTorClient.js b/scripts/packageTorClient.js index 74f3d1e5..79038806 100644 --- a/scripts/packageTorClient.js +++ b/scripts/packageTorClient.js @@ -67,7 +67,7 @@ const getOriginalManifest = (platform) => { return path.join('manifests', 'tor-client-updater', `tor-client-updater-${platform}-manifest.json`) } -const packageTorClient = (binary, endpoint, region, platform, key, +const packageTorClient = async (binary, endpoint, region, platform, key, publisherProofKey) => { const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `tor-client-updater-${platform}.pem`) const originalManifest = getOriginalManifest(platform) @@ -77,12 +77,18 @@ const packageTorClient = (binary, endpoint, region, platform, key, const stagingDir = path.join('build', 'tor-client-updater', platform) const crxFile = path.join('build', 'tor-client-updater', `tor-client-updater-${platform}.crx`) - util.getNextVersion(endpoint, region, id).then((version) => { - stageFiles(platform, torClient, version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) + + await util.prepareNextVersionCRX( + binary, + publisherProofKey, + endpoint, + region, + id, + stageFiles.bind(undefined, platform, torClient), + stagingDir, + crxFile, + privateKeyFile, + false) } const stageFiles = (platform, torClient, version, outputDir) => { @@ -120,13 +126,13 @@ if (fs.existsSync(commander.keyFile)) { throw new Error('Missing or invalid private key file/directory') } -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - packageTorClient(commander.binary, commander.endpoint, commander.region, +util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { + await packageTorClient(commander.binary, commander.endpoint, commander.region, 'darwin', keyParam, commander.publisherProofKey) - packageTorClient(commander.binary, commander.endpoint, commander.region, + await packageTorClient(commander.binary, commander.endpoint, commander.region, 'linux', keyParam, commander.publisherProofKey) - packageTorClient(commander.binary, commander.endpoint, commander.region, + await packageTorClient(commander.binary, commander.endpoint, commander.region, 'linux-arm64', keyParam, commander.publisherProofKey) - packageTorClient(commander.binary, commander.endpoint, commander.region, + await packageTorClient(commander.binary, commander.endpoint, commander.region, 'win32', keyParam, commander.publisherProofKey) }) diff --git a/scripts/packageTorPluggableTransports.js b/scripts/packageTorPluggableTransports.js index 08b9de85..3a198e85 100644 --- a/scripts/packageTorPluggableTransports.js +++ b/scripts/packageTorPluggableTransports.js @@ -45,7 +45,7 @@ const getOriginalManifest = (platform) => { return path.join('manifests', TOR_PLUGGABLE_TRANSPORTS_UPDATER, `${TOR_PLUGGABLE_TRANSPORTS_UPDATER}-${platform}-manifest.json`) } -const packageTorPluggableTransports = (binary, endpoint, region, platform, key, publisherProofKey) => { +const packageTorPluggableTransports = async (binary, endpoint, region, platform, key, publisherProofKey) => { const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `${TOR_PLUGGABLE_TRANSPORTS_UPDATER}-${platform}.pem`) const originalManifest = getOriginalManifest(platform) const parsedManifest = util.parseManifest(originalManifest) @@ -56,11 +56,18 @@ const packageTorPluggableTransports = (binary, endpoint, region, platform, key, const stagingDir = path.join('build', TOR_PLUGGABLE_TRANSPORTS_UPDATER, platform) const crxFile = path.join('build', TOR_PLUGGABLE_TRANSPORTS_UPDATER, `${TOR_PLUGGABLE_TRANSPORTS_UPDATER}-${platform}.crx`) - util.getNextVersion(endpoint, region, id).then((version) => { - stageFiles(platform, snowflake, obfs4, version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) + + await util.prepareNextVersionCRX( + binary, + publisherProofKey, + endpoint, + region, + id, + stageFiles.bind(undefined, platform, snowflake, obfs4), + stagingDir, + crxFile, + privateKeyFile, + false) } const stageFiles = (platform, snowflake, obfs4, version, outputDir) => { @@ -90,11 +97,11 @@ if (fs.existsSync(commander.keyFile)) { throw new Error('Missing or invalid private key file/directory') } -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - packageTorPluggableTransports(commander.binary, commander.endpoint, commander.region, +util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { + await packageTorPluggableTransports(commander.binary, commander.endpoint, commander.region, 'darwin', keyParam, commander.publisherProofKey) - packageTorPluggableTransports(commander.binary, commander.endpoint, commander.region, + await packageTorPluggableTransports(commander.binary, commander.endpoint, commander.region, 'linux', keyParam, commander.publisherProofKey) - packageTorPluggableTransports(commander.binary, commander.endpoint, commander.region, + await packageTorPluggableTransports(commander.binary, commander.endpoint, commander.region, 'win32', keyParam, commander.publisherProofKey) }) From 6211b571166fe450cf37efac25759bcb2ae0ff5e Mon Sep 17 00:00:00 2001 From: Anton Lazarev Date: Tue, 6 Feb 2024 15:42:59 -0800 Subject: [PATCH 3/5] use per-component classes to customize packaging behavior --- lib/adBlockRustUtils.js | 78 ++++++- lib/util.js | 24 +- scripts/generateAdBlockRustDataFiles.js | 142 +----------- scripts/generateManifestForRustAdblock.js | 58 +---- scripts/ntp-sponsored-images/package.js | 51 +++-- scripts/packageAdBlock.js | 216 +++++++++++++----- scripts/packageBraveAdsResourcesComponent.js | 60 ++--- scripts/packageBravePlayer.js | 61 ++--- scripts/packageComponent.js | 90 +++++--- scripts/packageIpfsDaemon.js | 51 +++-- scripts/packageLocalDataFiles.js | 78 ++++--- .../packageNTPBackgroundImagesComponent.js | 43 ++-- scripts/packageNTPSuperReferrerComponent.js | 45 ++-- ...geNTPSuperReferrerMappingTableComponent.js | 46 ++-- scripts/packagePlaylist.js | 42 ++-- scripts/packageTorClient.js | 65 +++--- scripts/packageTorPluggableTransports.js | 54 +++-- 17 files changed, 648 insertions(+), 556 deletions(-) diff --git a/lib/adBlockRustUtils.js b/lib/adBlockRustUtils.js index 74bb1363..ce1d5455 100644 --- a/lib/adBlockRustUtils.js +++ b/lib/adBlockRustUtils.js @@ -1,7 +1,8 @@ -import { Engine, FilterSet, uBlockResources } from 'adblock-rs' +import { Engine, FilterSet, RuleTypes, uBlockResources } from 'adblock-rs' +import fs from 'fs-extra' import path from 'path' -import { promises as fs } from 'fs' +import util from '../lib/util.js' const uBlockLocalRoot = 'submodules/uBlock' const uBlockWebAccessibleResources = path.join(uBlockLocalRoot, 'src/web_accessible_resources') @@ -50,9 +51,6 @@ const getListCatalog = lazyInit(async () => { // Legacy logic requires a distinction between default and regional lists. // This can be removed once DAT support is no longer needed by iOS. const isDefaultList = entry => entry.default_enabled && entry.hidden -const getDefaultLists = () => getListCatalog().then(catalog => { - return catalog.filter(isDefaultList) -}) const getRegionalLists = () => getListCatalog().then(catalog => { return catalog.filter(entry => !isDefaultList(entry)) }) @@ -126,6 +124,72 @@ const generateResourcesFile = async (outLocation) => { return fs.writeFile(outLocation, await generateResources(), 'utf8') } +// Removes Brave-specific scriptlet injections from non-Brave lists +const enforceBraveDirectives = (title, data) => { + if (!title || !title.startsWith('Brave ')) { + return data.split('\n').filter(line => { + const hasBraveScriptlet = line.indexOf('+js(brave-') >= 0 + if (hasBraveScriptlet) { + console.log('List ' + title + ' attempted to include brave-specific directive: ' + line) + } + return !hasBraveScriptlet + }).join('\n') + } else { + return data + } +} + +/** + * Parses the passed in filter rule data and serializes a data file to disk. + * + * @param filterRuleData An array of { format, data, includeRedirectUrls, ruleTypes } where format is one of `adblock-rust`'s supported filter parsing formats and data is a newline-separated list of such filters. + * includeRedirectUrls is a boolean: https://github.com/brave/adblock-rust/pull/184. We only support redirect URLs on filter lists we maintain and trust. + * ruleTypes was added with https://github.com/brave/brave-core-crx-packager/pull/298 and allows for { RuleTypes.ALL, RuleTypes.NETWORK_ONLY, RuleTypes.COSMETIC_ONLY } + * @param outputDATFilename The filename of the DAT file to create. + */ +const generateDataFileFromLists = (filterRuleData, outPath, defaultRuleType = RuleTypes.ALL) => { + const filterSet = new FilterSet(false) + for (let { title, format, data, includeRedirectUrls, ruleTypes } of filterRuleData) { + includeRedirectUrls = Boolean(includeRedirectUrls) + ruleTypes = ruleTypes || defaultRuleType + const parseOpts = { format, includeRedirectUrls, ruleTypes } + filterSet.addFilters(enforceBraveDirectives(title, data).split('\n'), parseOpts) + } + const client = new Engine(filterSet, true) + const arrayBuffer = client.serializeRaw() + fs.writeFileSync(outPath, Buffer.from(arrayBuffer)) +} + +/** + * Serializes the provided lists to disk in one file as `list.txt` under the given component subdirectory. + */ +const generatePlaintextListFromLists = (listBuffers, outPath) => { + const fullList = listBuffers.map(({ data, title }) => enforceBraveDirectives(title, data)).join('\n') + fs.writeFileSync(outPath, fullList) +} + +/** + * Returns a promise that resolves to the contents of each list from the entry's sources. + * Throws if any of the lists fail the sanity check. + */ +const downloadListsForEntry = (entry) => { + const lists = entry.sources + + const promises = [] + lists.forEach((l) => { + console.log(`${entry.langs} ${l.url}...`) + promises.push(util.fetchTextFromURL(l.url) + .then(data => ({ title: l.title || entry.title, format: l.format, data })) + .then(listBuffer => { + sanityCheckList(listBuffer) + return listBuffer + }) + ) + }) + + return Promise.all(promises) +} + /** * A list of requests that should not be blocked unless the list has some serious issue. * @@ -178,13 +242,15 @@ const sanityCheckList = ({ title, format, data }) => { } export { + downloadListsForEntry, regionalCatalogComponentId, regionalCatalogPubkey, resourcesComponentId, resourcesPubkey, sanityCheckList, + generateDataFileFromLists, + generatePlaintextListFromLists, generateResourcesFile, getListCatalog, - getDefaultLists, getRegionalLists } diff --git a/lib/util.js b/lib/util.js index adcd147c..239550d2 100644 --- a/lib/util.js +++ b/lib/util.js @@ -64,39 +64,47 @@ const fetchTextFromURL = (listURL) => { return p } -const prepareNextVersionCRX = async (binary, publisherProofKey, endpoint, region, componentId, stageFilesFn, stagingDir, outputCrx, privateKeyFile, localRun, contentHashFn, contentHashFile) => { +const prepareNextVersionCRX = async (binary, publisherProofKey, endpoint, region, componentDescriptor, privateKeyFile, localRun) => { + mkdirp.sync(componentDescriptor.stagingDir) + let contentHash - if (contentHashFn !== undefined) { - contentHash = contentHashFn() + if (componentDescriptor.contentHash !== undefined) { + try { + contentHash = componentDescriptor.contentHash() + } catch (_) { + // ignore failure if e.g. previous version does not exist yet + } } let version if (!localRun) { version = '1.0.0' } else { - version = await getNextVersion(endpoint, region, componentId, contentHash) + version = await getNextVersion(endpoint, region, componentDescriptor.componentId, contentHash) if (version === undefined) { - console.log(`content for ${componentId} was not updated, skipping!`) + console.log(`content for ${componentDescriptor.componentId} was not updated, skipping!`) return } } + const contentHashFile = componentDescriptor.contentHashFile + // Remove any existing `.contentHash` file for determinism if (contentHashFile !== undefined && fs.existsSync(contentHashFile)) { fs.unlinkSync(contentHashFile) } - stageFilesFn(version, stagingDir) + await componentDescriptor.stageFiles(version, componentDescriptor.stagingDir) if (!localRun) { - generateCRXFile(binary, outputCrx, privateKeyFile, publisherProofKey, stagingDir) + generateCRXFile(binary, componentDescriptor.crxFile, privateKeyFile, publisherProofKey, componentDescriptor.stagingDir) } if (contentHash !== undefined && contentHashFile !== undefined) { fs.writeFileSync(contentHashFile, contentHash) } - console.log(`Generated ${outputCrx} with version number ${version}`) + console.log(`Generated ${componentDescriptor.crxFile} with version number ${version}`) } const generateCRXFile = (binary, crxFile, privateKeyFile, publisherProofKey, diff --git a/scripts/generateAdBlockRustDataFiles.js b/scripts/generateAdBlockRustDataFiles.js index b46977c9..b1dc0a67 100644 --- a/scripts/generateAdBlockRustDataFiles.js +++ b/scripts/generateAdBlockRustDataFiles.js @@ -2,143 +2,5 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { - generateResourcesFile, - getListCatalog, - getDefaultLists, - getRegionalLists, - resourcesComponentId, - regionalCatalogComponentId, - sanityCheckList -} from '../lib/adBlockRustUtils.js' -import Sentry from '../lib/sentry.js' -import util from '../lib/util.js' -import path from 'path' -import fs from 'fs' - -/** - * Obtains the output path to store a file given the specied name and subdir - */ -const getOutPath = (outputFilename, outSubdir) => { - let outPath = path.join('build') - if (!fs.existsSync(outPath)) { - fs.mkdirSync(outPath) - } - outPath = path.join(outPath, 'ad-block-updater') - if (!fs.existsSync(outPath)) { - fs.mkdirSync(outPath) - } - outPath = path.join(outPath, outSubdir) - if (!fs.existsSync(outPath)) { - fs.mkdirSync(outPath) - } - return path.join(outPath, outputFilename) -} - -// Removes Brave-specific scriptlet injections from non-Brave lists -const enforceBraveDirectives = (title, data) => { - if (!title || !title.startsWith('Brave ')) { - return data.split('\n').filter(line => { - const hasBraveScriptlet = line.indexOf('+js(brave-') >= 0 - if (hasBraveScriptlet) { - console.log('List ' + title + ' attempted to include brave-specific directive: ' + line) - } - return !hasBraveScriptlet - }).join('\n') - } else { - return data - } -} - -/** - * Serializes the provided lists to disk in one file as `list.txt` under the given component subdirectory. - */ -const generatePlaintextListFromLists = (listBuffers, outSubdir) => { - const fullList = listBuffers.map(({ data, title }) => enforceBraveDirectives(title, data)).join('\n') - fs.writeFileSync(getOutPath('list.txt', outSubdir), fullList) -} - -/** - * Convenience function that generates component files for a given catalog entry. - * - * If any list source cannot be downloaded, the promise will resolve but the new files will _not_ be generated. - * - * @param entry the corresponding entry directly from one of Brave's list catalogs - * @param doIos boolean, whether or not filters for iOS should be created (currently only used by default list) - * @return a Promise which resolves upon completion - */ -const generateDataFilesForCatalogEntry = (entry) => { - const lists = entry.sources - - const promises = [] - lists.forEach((l) => { - console.log(`${entry.langs} ${l.url}...`) - promises.push(util.fetchTextFromURL(l.url) - .then(data => ({ title: l.title || entry.title, format: l.format, data })) - .then(listBuffer => { - sanityCheckList(listBuffer) - return listBuffer - }) - ) - }) - return Promise.all(promises) - .then( - listBuffers => generatePlaintextListFromLists(listBuffers, entry.list_text_component.component_id), - e => { - console.error(`Not publishing a new version of ${entry.title} due to failure downloading a source: ${e.message}`) - if (Sentry) { - Sentry.captureException(e, { level: 'warning' }) - } - } - ) -} - -/** - * Convenience function that generates a DAT file for each region, and writes - * the catalog of available regional lists to the default list directory and - * regional catalog component directory. - */ -const generateDataFilesForAllRegions = () => { - console.log('Processing per region list updates...') - return getRegionalLists().then(regions => { - return new Promise((resolve, reject) => { - const catalogString = JSON.stringify(regions) - fs.writeFileSync(getOutPath('regional_catalog.json', regionalCatalogComponentId), catalogString) - getListCatalog().then(listCatalog => { - const catalogString = JSON.stringify(listCatalog) - fs.writeFileSync(getOutPath('list_catalog.json', regionalCatalogComponentId), catalogString) - resolve() - }) - }).then(() => Promise.all(regions.map(region => - generateDataFilesForCatalogEntry(region) - ))) - }) -} - -const generateDataFilesForResourcesComponent = () => { - return generateResourcesFile(getOutPath('resources.json', resourcesComponentId)) -} - -const generateDataFilesForDefaultAdblock = () => getDefaultLists() - .then(defaultLists => Promise.all(defaultLists.map(list => generateDataFilesForCatalogEntry(list)))) - -generateDataFilesForDefaultAdblock() - .then(generateDataFilesForResourcesComponent) - .then(generateDataFilesForAllRegions) - .then(() => { - console.log('Thank you for updating the data files, don\'t forget to upload them too!') - }) - .catch((e) => { - console.error(`Something went wrong, aborting: ${e} ${e.stack} ${e.message}`) - process.exit(1) - }) - -process.on('uncaughtException', (err) => { - console.error('Caught exception:', err) - process.exit(1) -}) - -process.on('unhandledRejection', (err) => { - console.error('Unhandled rejection:', err) - process.exit(1) -}) +// TODO remove this after the Jenkins job is updated +console.log('Data files for adblock-rust are now generated as part of the package-ad-block job.') diff --git a/scripts/generateManifestForRustAdblock.js b/scripts/generateManifestForRustAdblock.js index 2a8ecaa6..0af1a7af 100644 --- a/scripts/generateManifestForRustAdblock.js +++ b/scripts/generateManifestForRustAdblock.js @@ -2,59 +2,5 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { promises as fs } from 'fs' -import path from 'path' - -import { getListCatalog, regionalCatalogComponentId, regionalCatalogPubkey, resourcesComponentId, resourcesPubkey } from '../lib/adBlockRustUtils.js' -import util from '../lib/util.js' - -const outPath = path.join('build', 'ad-block-updater') - -const generateManifestFile = async (name, base64PublicKey, subdir) => { - const manifest = '{\n' + - ' "description": "Brave Ad Block Updater extension",\n' + - ' "key": "' + base64PublicKey + '",\n' + - ' "manifest_version": 2,\n' + - ' "name": "Brave Ad Block Updater (' + name + ')",\n' + - ' "version": "0.0.0"\n' + - '}\n' - - const filePath = path.join(outPath, subdir, 'manifest.json') - return fs.writeFile(filePath, manifest) - .catch(e => console.warn('Skipped writing manifest for ' + name + ': ' + e.message)) -} - -const generateManifestFileForRegionalCatalog = - generateManifestFile.bind(null, 'Regional Catalog', regionalCatalogPubkey, regionalCatalogComponentId) - -const generateManifestFileForResources = - generateManifestFile.bind(null, 'Resources', resourcesPubkey, resourcesComponentId) - -const generateManifestFilesForAllLists = async () => { - const catalog = await getListCatalog() - return Promise.all(catalog.map(async entry => { - const title = util.escapeStringForJSON(entry.title) - await generateManifestFile(title + ' (plaintext)', entry.list_text_component.base64_public_key, entry.list_text_component.component_id) - })) -} - -generateManifestFileForRegionalCatalog() - .then(generateManifestFileForResources) - .then(generateManifestFilesForAllLists) - .then(() => { - console.log('Thank you for updating the data files, don\'t forget to upload them too!') - }) - .catch((e) => { - console.error(`Something went wrong, aborting: ${e}`) - process.exit(1) - }) - -process.on('uncaughtException', (err) => { - console.error('Caught exception:', err) - process.exit(1) -}) - -process.on('unhandledRejection', (err) => { - console.error('Unhandled rejection:', err) - process.exit(1) -}) +// TODO remove this after the Jenkins job is updated +console.log('Manifests for adblock-rust are now generated as part of the package-ad-block job.') diff --git a/scripts/ntp-sponsored-images/package.js b/scripts/ntp-sponsored-images/package.js index cdb71201..16a7aaa9 100644 --- a/scripts/ntp-sponsored-images/package.js +++ b/scripts/ntp-sponsored-images/package.js @@ -10,19 +10,13 @@ import path from 'path' import util from '../../lib/util.js' import params from './params.js' -const stageFiles = (locale, version, outputDir) => { - util.stageDir( - path.join(path.resolve(), 'build', 'ntp-sponsored-images', 'resources', locale, '/'), - getManifestPath(locale), - version, - outputDir) -} +const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-sponsored-images') -const generateManifestFile = (regionPlatform, componentData) => { +const generateManifestFile = (regionPlatform, publicKey) => { const manifestPath = getManifestPath(regionPlatform) const manifestContent = { description: `Brave NTP sponsored images component (${regionPlatform})`, - key: componentData.key, + key: publicKey, manifest_version: 2, name: 'Brave NTP sponsored images', version: '0.0.0' @@ -46,24 +40,44 @@ function getManifestPath (regionPlatform) { return path.join(getManifestsDir(), `${regionPlatform}-manifest.json`) } +class NtpSponsoredImages { + constructor (platformRegion, componentData) { + this.platformRegion = platformRegion + this.locale = componentData.locale + this.publicKey = componentData.key + this.componentId = componentData.id + + this.stagingDir = path.join(rootBuildDir, 'staging', this.platformRegion) + this.crxFile = path.join(rootBuildDir, 'output', `ntp-sponsored-images-${this.platformRegion}.crx`) + } + + privateKeyFromDir (keyDir) { + // Desktop private key file names do not have the -desktop suffix, but android has -android + return path.join(keyDir, `ntp-sponsored-images-${this.platformRegion.replace('-desktop', '')}.pem`) + } + + async stageFiles (version, outputDir) { + generateManifestFile(this.platformRegion, this.publicKey) + util.stageDir( + path.join(path.resolve(), 'build', 'ntp-sponsored-images', 'resources', this.locale, '/'), + getManifestPath(this.locale), + version, + outputDir) + } +} + const generateCRXFile = async (binary, endpoint, region, keyDir, platformRegion, componentData, publisherProofKey) => { - // Desktop private key file names do not have the -desktop suffix, but android has -android - const privateKeyFile = path.join(keyDir, `ntp-sponsored-images-${platformRegion.replace('-desktop', '')}.pem`) - const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-sponsored-images') + const descriptor = new NtpSponsoredImages(platformRegion, componentData) - const stagingDir = path.join(rootBuildDir, 'staging', platformRegion) - const crxFile = path.join(rootBuildDir, 'output', `ntp-sponsored-images-${platformRegion}.crx`) + const privateKeyFile = descriptor.privateKeyFromDir(keyDir) await util.prepareNextVersionCRX( binary, publisherProofKey, endpoint, region, - componentData.id, - stageFiles.bind(undefined, platformRegion), - stagingDir, - crxFile, + descriptor, privateKeyFile, false) } @@ -89,7 +103,6 @@ const targetComponents = params.getTargetComponents(commander.targetRegions, com util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { for (const platformRegion of Object.keys(targetComponents)) { const componentData = targetComponents[platformRegion] - generateManifestFile(platformRegion, componentData) await generateCRXFile(commander.binary, commander.endpoint, commander.region, keyDir, platformRegion, componentData, commander.publisherProofKey) diff --git a/scripts/packageAdBlock.js b/scripts/packageAdBlock.js index ac5e66e2..3c8beefb 100644 --- a/scripts/packageAdBlock.js +++ b/scripts/packageAdBlock.js @@ -9,49 +9,171 @@ import commander from 'commander' import fs from 'fs-extra' import path from 'path' +import Sentry from '../lib/sentry.js' import util from '../lib/util.js' -import { getListCatalog, regionalCatalogComponentId, resourcesComponentId } from '../lib/adBlockRustUtils.js' - -function stageFiles (version, outputDir) { - // ad-block components are already written in the output directory - // so we don't need to stage anything - const originalManifest = path.join(outputDir, 'manifest.json') - // note - in-place manifest replacement, unlike other components - util.copyManifestWithVersion(originalManifest, outputDir, version) -} +import { + downloadListsForEntry, + generatePlaintextListFromLists, + generateResourcesFile, + getListCatalog, + getRegionalLists, + resourcesComponentId, + resourcesPubkey, + regionalCatalogComponentId, + regionalCatalogPubkey +} from '../lib/adBlockRustUtils.js' const getOriginalManifest = (componentSubdir) => { const manifestsDir = path.join('build', 'ad-block-updater') return path.join(manifestsDir, componentSubdir, 'manifest.json') } -const processComponent = async (binary, endpoint, region, keyDir, - publisherProofKey, localRun, componentSubdir) => { - const originalManifest = getOriginalManifest(componentSubdir) - const parsedManifest = util.parseManifest(originalManifest) - const id = util.getIDFromBase64PublicKey(parsedManifest.key) - - const crxName = `ad-block-updater-${componentSubdir}` - const privateKeyFile = localRun ? undefined : path.join(keyDir, `${crxName}.pem`) - const stagingDir = path.join('build', 'ad-block-updater', componentSubdir) - const crxFile = path.join('build', 'ad-block-updater', `${crxName}.crx`) - const contentHashFile = path.join('build', 'ad-block-updater', `${crxName}.contentHash`) - - const contentHashFn = () => { - let fileToHash - if (componentSubdir === regionalCatalogComponentId) { - fileToHash = 'regional_catalog.json' - } else if (componentSubdir === resourcesComponentId) { - fileToHash = 'resources.json' - } else { - fileToHash = 'list.txt' - } - if (fileToHash !== undefined) { - const contentFile = path.join('build', 'ad-block-updater', componentSubdir, fileToHash) - return util.generateSHA256HashOfFile(contentFile) - } else { - return undefined +const generateManifestFile = async (name, base64PublicKey, componentSubdir) => { + const manifest = '{\n' + + ' "description": "Brave Ad Block Updater extension",\n' + + ' "key": "' + base64PublicKey + '",\n' + + ' "manifest_version": 2,\n' + + ' "name": "Brave Ad Block Updater (' + name + ')",\n' + + ' "version": "0.0.0"\n' + + '}\n' + + const filePath = getOriginalManifest(componentSubdir) + return fs.writeFile(filePath, manifest) + .catch(e => console.warn('Skipped writing manifest for ' + name + ': ' + e.message)) +} + +// Serves plaintext filter lists +class AdblockList { + constructor (catalogEntry) { + this.catalogEntry = catalogEntry + this.componentId = catalogEntry.list_text_component.component_id + + this.crxName = `ad-block-updater-${this.componentId}` + this.stagingDir = path.join('build', 'ad-block-updater', this.componentId) + this.crxFile = path.join('build', 'ad-block-updater', `${this.crxName}.crx`) + this.contentHashFile = path.join('build', 'ad-block-updater', `${this.crxName}.contentHash`) + } + + contentHash () { + const contentFile = path.join('build', 'ad-block-updater', this.componentId, 'list.txt') + return util.generateSHA256HashOfFile(contentFile) + } + + privateKeyFromDir (keyDir) { + return path.join(keyDir, `${this.crxName}.pem`) + } + + async stageFiles (version, outputDir) { + await downloadListsForEntry(this.catalogEntry).then( + (listBuffers) => { + generatePlaintextListFromLists(listBuffers, path.join(outputDir, 'list.txt')) + }, + e => { + console.error(`Not publishing a new version of ${this.catalogEntry.title} due to failure downloading a source: ${e.message}`) + if (Sentry) { + Sentry.captureException(e, { level: 'warning' }) + } + } + ) + + const title = util.escapeStringForJSON(this.catalogEntry.title) + if (this.catalogEntry.list_text_component) { + await generateManifestFile(title + ' (plaintext)', this.catalogEntry.list_text_component.base64_public_key, this.componentId) } + + const files = [ + { path: getOriginalManifest(this.componentId) }, + { path: path.join(outputDir, 'list.txt') } + ] + util.stageFiles(files, version, outputDir) + } +} + +// Serves the catalog of available filter lists +// Currently there are two files: +// - `regional_catalog.json` is the older version +// - `list_catalog.json` is the newer version, containing additional list metadata +class AdblockCatalog { + constructor () { + this.componentId = regionalCatalogComponentId + + this.crxName = `ad-block-updater-${this.componentId}` + this.stagingDir = path.join('build', 'ad-block-updater', this.componentId) + this.crxFile = path.join('build', 'ad-block-updater', `${this.crxName}.crx`) + this.contentHashFile = path.join('build', 'ad-block-updater', `${this.crxName}.contentHash`) + } + + contentHash () { + // TODO update to `list_catalog.json` when `regional_catalog.json` is no longer used + const contentFile = path.join('build', 'ad-block-updater', this.componentId, 'regional_catalog.json') + return util.generateSHA256HashOfFile(contentFile) + } + + privateKeyFromDir (keyDir) { + return path.join(keyDir, `${this.crxName}.pem`) + } + + async stageFiles (version, outputDir) { + const regions = await getRegionalLists() + const regionalCatalogString = JSON.stringify(regions) + const regionalCatalogPath = path.join(outputDir, 'regional_catalog.json') + fs.writeFileSync(regionalCatalogPath, regionalCatalogString) + + const listCatalog = await getListCatalog() + const listCatalogString = JSON.stringify(listCatalog) + const listCatalogPath = path.join(outputDir, 'list_catalog.json') + fs.writeFileSync(listCatalogPath, listCatalogString) + + await generateManifestFile('Regional Catalog', regionalCatalogPubkey, regionalCatalogComponentId) + + const files = [ + { path: getOriginalManifest(this.componentId) }, + { path: regionalCatalogPath }, + { path: listCatalogPath } + ] + util.stageFiles(files, version, outputDir) + } +} + +// Serves the data for adblock-rust's resource replacements and scriptlets. +class AdblockResources { + constructor () { + this.componentId = resourcesComponentId + + this.crxName = `ad-block-updater-${this.componentId}` + this.stagingDir = path.join('build', 'ad-block-updater', this.componentId) + this.crxFile = path.join('build', 'ad-block-updater', `${this.crxName}.crx`) + this.contentHashFile = path.join('build', 'ad-block-updater', `${this.crxName}.contentHash`) + } + + contentHash () { + const contentFile = path.join('build', 'ad-block-updater', this.componentId, 'resources.json') + return util.generateSHA256HashOfFile(contentFile) + } + + privateKeyFromDir (keyDir) { + return path.join(keyDir, `${this.crxName}.pem`) + } + + async stageFiles (version, outputDir) { + const resourcesPath = path.join(outputDir, 'resources.json') + await generateResourcesFile(resourcesPath) + + await generateManifestFile('Resources', resourcesPubkey, resourcesComponentId) + + const files = [ + { path: getOriginalManifest(this.componentId) }, + { path: resourcesPath } + ] + util.stageFiles(files, version, outputDir) + } +} + +const processComponent = async (binary, endpoint, region, keyParam, + publisherProofKey, localRun, descriptor) => { + let privateKeyFile = '' + if (!localRun) { + privateKeyFile = !fs.lstatSync(keyParam).isDirectory() ? keyParam : descriptor.privateKeyFromDir(keyParam) } await util.prepareNextVersionCRX( @@ -59,31 +181,23 @@ const processComponent = async (binary, endpoint, region, keyDir, publisherProofKey, endpoint, region, - id, - stageFiles, - stagingDir, - crxFile, + descriptor, privateKeyFile, - localRun, - contentHashFn, - contentHashFile) + localRun) } -const getComponentList = async () => { +const getAdblockComponentDescriptors = async () => { const output = [ - regionalCatalogComponentId, - resourcesComponentId + new AdblockCatalog(), + new AdblockResources() ] - const catalog = await getListCatalog() - catalog.forEach(entry => { - output.push(entry.list_text_component.component_id) - }) + output.push(...(await getListCatalog()).map(entry => new AdblockList(entry))) return output } const processJob = async (commander, keyDir) => { - (await getComponentList()) - .forEach(processComponent.bind(null, commander.binary, commander.endpoint, + (await getAdblockComponentDescriptors()) + .map(processComponent.bind(null, commander.binary, commander.endpoint, commander.region, keyDir, commander.publisherProofKey, commander.localRun)) diff --git a/scripts/packageBraveAdsResourcesComponent.js b/scripts/packageBraveAdsResourcesComponent.js index 026087c2..45113a12 100644 --- a/scripts/packageBraveAdsResourcesComponent.js +++ b/scripts/packageBraveAdsResourcesComponent.js @@ -233,12 +233,16 @@ const getComponentDataList = () => { ] } -const stageFiles = (locale, version, outputDir) => { - util.stageDir( - path.join(path.resolve(), 'build', 'user-model-installer', 'resources', locale, '/'), - getOriginalManifest(locale), - version, - outputDir) +const rootBuildDir = path.join(path.resolve(), 'build', 'user-model-installer') + +const getManifestsDir = () => { + const targetResourceDir = path.join(path.resolve(), 'build', 'user-model-installer', 'manifiest-files') + mkdirp.sync(targetResourceDir) + return targetResourceDir +} + +const getOriginalManifest = (locale) => { + return path.join(getManifestsDir(), `${locale}-manifest.json`) } const generateManifestFile = (componentData) => { @@ -253,38 +257,41 @@ const generateManifestFile = (componentData) => { fs.writeFileSync(manifestFile, JSON.stringify(manifestContent)) } -const generateManifestFiles = () => { - getComponentDataList().forEach(generateManifestFile) -} +class BraveAdsResourcesComponent { + constructor (componentData) { + this.locale = componentData.locale + this.publicKey = componentData.key + this.componentId = componentData.id + this.stagingDir = path.join(rootBuildDir, 'staging', this.locale) + this.crxFile = path.join(rootBuildDir, 'output', `user-model-installer-${this.locale}.crx`) + } -const getManifestsDir = () => { - const targetResourceDir = path.join(path.resolve(), 'build', 'user-model-installer', 'manifiest-files') - mkdirp.sync(targetResourceDir) - return targetResourceDir -} + privateKeyFromDir (keyDir) { + return path.join(keyDir, `user-model-installer-${this.locale}.pem`) + } -const getOriginalManifest = (locale) => { - return path.join(getManifestsDir(), `${locale}-manifest.json`) + async stageFiles (version, outputDir) { + generateManifestFile(this.componentData) + util.stageDir( + path.join(path.resolve(), 'build', 'user-model-installer', 'resources', this.locale, '/'), + getOriginalManifest(this.locale), + version, + outputDir) + } } const generateCRXFile = async (binary, endpoint, region, keyDir, publisherProofKey, componentData) => { - const locale = componentData.locale - const privateKeyFile = path.join(keyDir, `user-model-installer-${locale}.pem`) - const rootBuildDir = path.join(path.resolve(), 'build', 'user-model-installer') + const descriptor = new BraveAdsResourcesComponent(componentData) - const stagingDir = path.join(rootBuildDir, 'staging', locale) - const crxFile = path.join(rootBuildDir, 'output', `user-model-installer-${locale}.crx`) + const privateKeyFile = descriptor.privateKeyFromDir(keyDir) - util.prepareNextVersionCRX( + await util.prepareNextVersionCRX( binary, publisherProofKey, endpoint, region, - componentData.id, - stageFiles.bind(undefined, locale), - stagingDir, - crxFile, + descriptor, privateKeyFile, false) } @@ -304,7 +311,6 @@ if (fs.existsSync(commander.keysDirectory)) { } util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - generateManifestFiles() await Promise.all(getComponentDataList().map( generateCRXFile.bind(null, commander.binary, commander.endpoint, commander.region, keyDir, diff --git a/scripts/packageBravePlayer.js b/scripts/packageBravePlayer.js index cebc3b58..66d5d822 100644 --- a/scripts/packageBravePlayer.js +++ b/scripts/packageBravePlayer.js @@ -19,44 +19,45 @@ const getOriginalManifest = () => { return path.join('component-data', 'brave-player', 'manifest.json') } -const stageFiles = (version, outputDir) => { - util.stageDir(path.join('component-data', 'brave-player'), getOriginalManifest(), version, outputDir) -} +class BravePlayer { + constructor () { + const originalManifest = getOriginalManifest() + const parsedManifest = util.parseManifest(originalManifest) + this.componentId = util.getIDFromBase64PublicKey(parsedManifest.key) -const postNextVersionWork = (key, publisherProofKey, binary, localRun, version) => { - const stagingDir = path.join('build', 'brave-player') - const crxOutputDir = path.join('build') - const crxFile = path.join(crxOutputDir, 'brave-player.crx') - let privateKeyFile = '' - if (!localRun) { - privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, 'brave-player.pem') + this.stagingDir = path.join('build', 'brave-player') + this.crxFile = path.join('build', 'brave-player.crx') } - stageFiles(version, stagingDir) - if (!localRun) { - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) + + privateKeyFromDir (keyDir) { + return path.join(keyDir, 'brave-player.pem') + } + + async stageFiles (version, outputDir) { + util.stageDir(path.join('component-data', 'brave-player'), getOriginalManifest(), version, outputDir) } - console.log(`Generated ${crxFile} with version number ${version}`) } -const processDATFile = (binary, endpoint, region, key, publisherProofKey, localRun) => { - const originalManifest = getOriginalManifest() - const parsedManifest = util.parseManifest(originalManifest) - const id = util.getIDFromBase64PublicKey(parsedManifest.key) +const packageBravePlayer = async (binary, endpoint, region, key, publisherProofKey, localRun) => { + const descriptor = new BravePlayer() + let privateKeyFile = '' if (!localRun) { - util.getNextVersion(endpoint, region, id).then((version) => { - postNextVersionWork(key, publisherProofKey, - binary, localRun, version) - }) - } else { - postNextVersionWork(key, publisherProofKey, - binary, localRun, '1.0.0') + privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : descriptor.privateKeyFromDir(key) } + + await util.prepareNextVersionCRX( + binary, + publisherProofKey, + endpoint, + region, + descriptor, + privateKeyFile, + localRun) } -const processJob = (commander, keyParam) => { - processDATFile(commander.binary, commander.endpoint, commander.region, +const processJob = async (commander, keyParam) => { + await packageBravePlayer(commander.binary, commander.endpoint, commander.region, keyParam, commander.publisherProofKey, commander.localRun) } @@ -82,8 +83,8 @@ if (!commander.localRun) { } if (!commander.localRun) { - util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - processJob(commander, keyParam) + util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { + await processJob(commander, keyParam) }) } else { processJob(commander, keyParam) diff --git a/scripts/packageComponent.js b/scripts/packageComponent.js index 655c9760..a261f287 100644 --- a/scripts/packageComponent.js +++ b/scripts/packageComponent.js @@ -10,68 +10,82 @@ import fs from 'fs-extra' import path from 'path' import util from '../lib/util.js' -const stageFiles = (componentType, version, outputDir) => { - util.stageDir(getPackageDirByComponentType(componentType), getOriginalManifest(componentType), version, outputDir) +const getOriginalManifest = (packageDir) => { + return path.join(packageDir, 'manifest.json') +} - if (componentType === 'wallet-data-files-updater') { - fs.unlinkSync(path.join(outputDir, 'package.json')) +class EthereumRemoteClient { + constructor () { + const originalManifest = getOriginalManifest(this.packageDir) + const parsedManifest = util.parseManifest(originalManifest) + this.componentId = util.getIDFromBase64PublicKey(parsedManifest.key) } -} -const validComponentTypes = [ - 'ethereum-remote-client', - 'wallet-data-files-updater' -] - -const getPackageNameByComponentType = (componentType) => { - switch (componentType) { - case 'ethereum-remote-client': - return componentType - case 'wallet-data-files-updater': - return 'brave-wallet-lists' - default: - // shouldn't be possible to get here - return null + componentType = 'ethereum-remote-client' + packageDir = path.join('node_modules', 'ethereum-remote-client') + + stagingDir = path.join('build', this.componentType) + crxFile = path.join(this.stagingDir, `${this.componentType}.crx`) + + privateKeyFromDir (keyDir) { + return path.join(keyDir, `${this.componentType}.pem`) + } + + async stageFiles (version, outputDir) { + util.stageDir(this.packageDir, getOriginalManifest(this.packageDir), version, outputDir) } } -const getPackageDirByComponentType = (componentType) => { - return path.join('node_modules', getPackageNameByComponentType(componentType)) -} -const getOriginalManifest = (componentType) => { - return path.join(getPackageDirByComponentType(componentType), 'manifest.json') +class WalletDataFilesUpdater { + constructor () { + const originalManifest = getOriginalManifest(this.packageDir) + const parsedManifest = util.parseManifest(originalManifest) + this.componentId = util.getIDFromBase64PublicKey(parsedManifest.key) + } + + componentType = 'wallet-data-files-updater' + packageDir = path.join('node_modules', 'brave-wallet-lists') + + stagingDir = path.join('build', this.componentType) + crxFile = path.join(this.stagingDir, `${this.componentType}.crx`) + + privateKeyFromDir (keyDir) { + return path.join(keyDir, `${this.componentType}.pem`) + } + + async stageFiles (version, outputDir) { + util.stageDir(this.packageDir, getOriginalManifest(this.packageDir), version, outputDir) + fs.unlinkSync(path.join(outputDir, 'package.json')) + } } const generateCRXFile = async (binary, endpoint, region, componentType, key, publisherProofKey, localRun) => { - const originalManifest = getOriginalManifest(componentType) - const parsedManifest = util.parseManifest(originalManifest) - const id = util.getIDFromBase64PublicKey(parsedManifest.key) + let descriptor + if (componentType === 'ethereum-remote-client') { + descriptor = new EthereumRemoteClient() + } else if (componentType === 'wallet-data-files-updater') { + descriptor = new WalletDataFilesUpdater() + } else { + throw new Error('Unrecognized component extension type: ' + commander.type) + } let privateKeyFile = '' if (!localRun) { - privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `${componentType}.pem`) + privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : descriptor.privateKeyFromDir(key) } - const stagingDir = path.join('build', componentType) - const crxFile = path.join(stagingDir, `${componentType}.crx`) await util.prepareNextVersionCRX( binary, publisherProofKey, endpoint, region, - id, - stageFiles.bind(undefined, componentType), - stagingDir, - crxFile, + descriptor, privateKeyFile, localRun) } const processJob = async (commander, keyParam) => { - if (!validComponentTypes.includes(commander.type)) { - throw new Error('Unrecognized component extension type: ' + commander.type) - } await generateCRXFile(commander.binary, commander.endpoint, commander.region, commander.type, keyParam, commander.publisherProofKey, @@ -84,7 +98,7 @@ util.addCommonScriptOptions( commander .option('-d, --keys-directory ', 'directory containing private keys for signing crx files') .option('-f, --key-file ', 'private key file for signing crx', 'key.pem') - .option('-t, --type ', 'component extension type', /^(local-data-files-updater|ethereum-remote-client|wallet-data-files-updater)$/i) + .option('-t, --type ', 'component extension type', /^(ethereum-remote-client|wallet-data-files-updater)$/i) .option('-l, --local-run', 'Runs updater job without connecting anywhere remotely')) .parse(process.argv) diff --git a/scripts/packageIpfsDaemon.js b/scripts/packageIpfsDaemon.js index 14d41631..ba7c8a26 100644 --- a/scripts/packageIpfsDaemon.js +++ b/scripts/packageIpfsDaemon.js @@ -22,39 +22,50 @@ const getOriginalManifest = (platform) => { return path.join('manifests', 'ipfs-daemon-updater', `ipfs-daemon-updater-${platform}-manifest.json`) } +class IpfsDaemon { + constructor (os, arch) { + this.os = os + this.arch = arch + this.platform = `${this.os}-${this.arch}` + + const originalManifest = getOriginalManifest(this.platform) + const parsedManifest = util.parseManifest(originalManifest) + this.componentId = util.getIDFromBase64PublicKey(parsedManifest.key) + + this.stagingDir = path.join('build', 'ipfs-daemon-updater', this.platform) + this.crxFile = path.join('build', 'ipfs-daemon-updater', `ipfs-daemon-updater-${this.platform}.crx`) + } + + privateKeyFromDir (keyDir) { + return path.join(keyDir, `ipfs-daemon-updater-${this.platform}.pem`) + } + + async stageFiles (version, outputDir) { + const ipfsDaemon = getIpfsDaemonPath(this.os, this.arch) + const files = [ + { path: getOriginalManifest(this.platform), outputName: 'manifest.json' }, + { path: ipfsDaemon } + ] + util.stageFiles(files, version, outputDir) + } +} + const packageIpfsDaemon = async (binary, endpoint, region, os, arch, key, publisherProofKey) => { - const platform = `${os}-${arch}` - const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `ipfs-daemon-updater-${platform}.pem`) - const originalManifest = getOriginalManifest(platform) - const parsedManifest = util.parseManifest(originalManifest) - const id = util.getIDFromBase64PublicKey(parsedManifest.key) - const ipfsDaemon = getIpfsDaemonPath(os, arch) + const descriptor = new IpfsDaemon(os, arch) - const stagingDir = path.join('build', 'ipfs-daemon-updater', platform) - const crxFile = path.join('build', 'ipfs-daemon-updater', `ipfs-daemon-updater-${platform}.crx`) + const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : descriptor.privateKeyFromDir(key) await util.prepareNextVersionCRX( binary, publisherProofKey, endpoint, region, - id, - stageFiles.bind(undefined, platform, ipfsDaemon), - stagingDir, - crxFile, + descriptor, privateKeyFile, false) } -const stageFiles = (platform, ipfsDaemon, version, outputDir) => { - const files = [ - { path: getOriginalManifest(platform), outputName: 'manifest.json' }, - { path: ipfsDaemon } - ] - util.stageFiles(files, version, outputDir) -} - util.installErrorHandlers() util.addCommonScriptOptions( diff --git a/scripts/packageLocalDataFiles.js b/scripts/packageLocalDataFiles.js index 4b641d80..9e8ddf76 100644 --- a/scripts/packageLocalDataFiles.js +++ b/scripts/packageLocalDataFiles.js @@ -12,58 +12,66 @@ import path from 'path' import recursive from 'recursive-readdir-sync' import util from '../lib/util.js' +const componentType = 'local-data-files-updater' +const datFileName = 'default' + const getOriginalManifest = () => { return path.join('manifests', 'local-data-files-updater', 'default-manifest.json') } -const stageFiles = (version, outputDir) => { - const datFileVersion = '1' - const files = [ - { path: getOriginalManifest(), outputName: 'manifest.json' }, - { path: path.join('brave-lists', 'debounce.json'), outputName: path.join(datFileVersion, 'debounce.json') }, - { path: path.join('brave-lists', 'request-otr.json'), outputName: path.join(datFileVersion, 'request-otr.json') }, - { path: path.join('brave-lists', 'clean-urls.json'), outputName: path.join(datFileVersion, 'clean-urls.json') }, - { path: path.join('brave-lists', 'https-upgrade-exceptions-list.txt'), outputName: path.join(datFileVersion, 'https-upgrade-exceptions-list.txt') }, - { path: path.join('brave-lists', 'localhost-permission-allow-list.txt'), outputName: path.join(datFileVersion, 'localhost-permission-allow-list.txt') } - ].concat( - recursive(path.join('node_modules', 'brave-site-specific-scripts', 'dist')).map(f => { - let outputDatDir = datFileVersion - const index = f.indexOf('/dist/') - let baseDir = f.substring(index + '/dist/'.length) - baseDir = baseDir.substring(0, baseDir.lastIndexOf('/')) - outputDatDir = path.join(outputDatDir, baseDir) - mkdirp.sync(path.join(outputDir, outputDatDir)) - return { - path: f, - outputName: path.join(outputDatDir, path.parse(f).base) - } - })) - util.stageFiles(files, version, outputDir) +class LocalDataFiles { + stagingDir = path.join('build', componentType, datFileName) + crxFile = path.join('build', componentType, `${componentType}-${datFileName}.crx`) + componentId = (() => { + const originalManifest = getOriginalManifest() + const parsedManifest = util.parseManifest(originalManifest) + return util.getIDFromBase64PublicKey(parsedManifest.key) + })() + + privateKeyFromDir (keyDir) { + return path.join(keyDir, `${componentType}-${datFileName}.pem`) + } + + async stageFiles (version, outputDir) { + const datFileVersion = '1' + const files = [ + { path: getOriginalManifest(), outputName: 'manifest.json' }, + { path: path.join('brave-lists', 'debounce.json'), outputName: path.join(datFileVersion, 'debounce.json') }, + { path: path.join('brave-lists', 'request-otr.json'), outputName: path.join(datFileVersion, 'request-otr.json') }, + { path: path.join('brave-lists', 'clean-urls.json'), outputName: path.join(datFileVersion, 'clean-urls.json') }, + { path: path.join('brave-lists', 'https-upgrade-exceptions-list.txt'), outputName: path.join(datFileVersion, 'https-upgrade-exceptions-list.txt') }, + { path: path.join('brave-lists', 'localhost-permission-allow-list.txt'), outputName: path.join(datFileVersion, 'localhost-permission-allow-list.txt') } + ].concat( + recursive(path.join('node_modules', 'brave-site-specific-scripts', 'dist')).map(f => { + let outputDatDir = datFileVersion + const index = f.indexOf('/dist/') + let baseDir = f.substring(index + '/dist/'.length) + baseDir = baseDir.substring(0, baseDir.lastIndexOf('/')) + outputDatDir = path.join(outputDatDir, baseDir) + mkdirp.sync(path.join(outputDir, outputDatDir)) + return { + path: f, + outputName: path.join(outputDatDir, path.parse(f).base) + } + })) + util.stageFiles(files, version, outputDir) + } } const generateCRXFile = async (binary, endpoint, region, key, publisherProofKey, localRun) => { - const originalManifest = getOriginalManifest() - const parsedManifest = util.parseManifest(originalManifest) - const id = util.getIDFromBase64PublicKey(parsedManifest.key) + const descriptor = new LocalDataFiles() - const componentType = 'local-data-files-updater' - const datFileName = 'default' let privateKeyFile = '' if (!localRun) { - privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `${componentType}-${datFileName}.pem`) + privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : descriptor.privateKeyFromDir(key) } - const stagingDir = path.join('build', componentType, datFileName) - const crxFile = path.join('build', componentType, `${componentType}-${datFileName}.crx`) await util.prepareNextVersionCRX( binary, publisherProofKey, endpoint, region, - id, - stageFiles, - stagingDir, - crxFile, + descriptor, privateKeyFile, localRun) } diff --git a/scripts/packageNTPBackgroundImagesComponent.js b/scripts/packageNTPBackgroundImagesComponent.js index 12f11f5d..1ae7b63e 100644 --- a/scripts/packageNTPBackgroundImagesComponent.js +++ b/scripts/packageNTPBackgroundImagesComponent.js @@ -8,15 +8,12 @@ import path from 'path' import util from '../lib/util.js' import ntpUtil from '../lib/ntpUtil.js' +const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-background-images') + const getOriginalManifest = () => { return path.join(path.resolve(), 'build', 'ntp-background-images', 'ntp-background-images-manifest.json') } -const stageFiles = util.stageDir.bind( - undefined, - path.join(path.resolve(), 'build', 'ntp-background-images', 'resources'), - getOriginalManifest()) - const generateManifestFile = (publicKey) => { const manifestFile = getOriginalManifest() const manifestContent = { @@ -29,22 +26,36 @@ const generateManifestFile = (publicKey) => { fs.writeFileSync(manifestFile, JSON.stringify(manifestContent)) } -const generateCRXFile = async (binary, endpoint, region, componentID, privateKeyFile, - publisherProofKey) => { - const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-background-images') +class NTPBackgroundImagesComponent { + constructor (privateKeyFile) { + const [publicKey, componentId] = ntpUtil.generatePublicKeyAndID(privateKeyFile) + this.publicKey = publicKey + this.componentId = componentId + } + + stagingDir = path.join(rootBuildDir, 'staging') + crxFile = path.join(rootBuildDir, 'output', 'ntp-background-images.crx') - const stagingDir = path.join(rootBuildDir, 'staging') - const crxFile = path.join(rootBuildDir, 'output', 'ntp-background-images.crx') + async stageFiles (version, outputDir) { + generateManifestFile(this.publicKey) + util.stageDir( + path.join(path.resolve(), 'build', 'ntp-background-images', 'resources'), + getOriginalManifest(), + version, + outputDir) + } +} + +const generateCRXFile = async (binary, endpoint, region, privateKeyFile, + publisherProofKey) => { + const descriptor = new NTPBackgroundImagesComponent(privateKeyFile) await util.prepareNextVersionCRX( binary, publisherProofKey, endpoint, region, - componentID, - stageFiles, - stagingDir, - crxFile, + descriptor, privateKeyFile, false) } @@ -64,8 +75,6 @@ if (fs.existsSync(commander.key)) { } util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - const [publicKey, componentID] = ntpUtil.generatePublicKeyAndID(privateKeyFile) - generateManifestFile(publicKey) await generateCRXFile(commander.binary, commander.endpoint, commander.region, - componentID, privateKeyFile, commander.publisherProofKey) + privateKeyFile, commander.publisherProofKey) }) diff --git a/scripts/packageNTPSuperReferrerComponent.js b/scripts/packageNTPSuperReferrerComponent.js index 3594786d..f868cd10 100644 --- a/scripts/packageNTPSuperReferrerComponent.js +++ b/scripts/packageNTPSuperReferrerComponent.js @@ -8,13 +8,7 @@ import path from 'path' import util from '../lib/util.js' import ntpUtil from '../lib/ntpUtil.js' -const stageFiles = (superReferrerName, version, outputDir) => { - util.stageDir( - path.join(path.resolve(), 'build', 'ntp-super-referrer', 'resources', superReferrerName, '/'), - getOriginalManifest(superReferrerName), - version, - outputDir) -} +const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer') const generateManifestFile = (superReferrerName, publicKey) => { const manifestFile = getOriginalManifest(superReferrerName) @@ -32,22 +26,36 @@ const getOriginalManifest = (superReferrerName) => { return path.join(path.resolve(), 'build', 'ntp-super-referrer', `${superReferrerName}-manifest.json`) } -const generateCRXFile = async (binary, endpoint, region, superReferrerName, - componentID, privateKeyFile, publisherProofKey) => { - const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer') +class NTPSuperReferrerComponent { + constructor (superReferrerName, privateKeyFile) { + const [publicKey, componentId] = ntpUtil.generatePublicKeyAndID(privateKeyFile) + this.publicKey = publicKey + this.componentId = componentId + this.superReferrerName = superReferrerName + this.stagingDir = path.join(rootBuildDir, 'staging', this.superReferrerName) + this.crxFile = path.join(rootBuildDir, 'output', `ntp-super-referrer-${this.superReferrerName}.crx`) + } - const stagingDir = path.join(rootBuildDir, 'staging', superReferrerName) - const crxFile = path.join(rootBuildDir, 'output', `ntp-super-referrer-${superReferrerName}.crx`) + async stageFiles (version, outputDir) { + generateManifestFile(this.superReferrerName, this.publicKey) + util.stageDir( + path.join(path.resolve(), 'build', 'ntp-super-referrer', 'resources', this.superReferrerName, '/'), + getOriginalManifest(this.superReferrerName), + version, + outputDir) + } +} + +const generateCRXFile = async (binary, endpoint, region, superReferrerName, + privateKeyFile, publisherProofKey) => { + const descriptor = new NTPSuperReferrerComponent(superReferrerName, privateKeyFile) await util.prepareNextVersionCRX( binary, publisherProofKey, endpoint, region, - componentID, - stageFiles.bind(undefined, superReferrerName), - stagingDir, - crxFile, + descriptor, privateKeyFile, false) } @@ -68,9 +76,6 @@ if (fs.existsSync(commander.key)) { } util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - const [publicKey, componentID] = ntpUtil.generatePublicKeyAndID(privateKeyFile) - generateManifestFile(commander.superReferrerName, publicKey) await generateCRXFile(commander.binary, commander.endpoint, commander.region, - commander.superReferrerName, componentID, privateKeyFile, - commander.publisherProofKey) + commander.superReferrerName, privateKeyFile, commander.publisherProofKey) }) diff --git a/scripts/packageNTPSuperReferrerMappingTableComponent.js b/scripts/packageNTPSuperReferrerMappingTableComponent.js index 3dc47c0e..014a9cbc 100644 --- a/scripts/packageNTPSuperReferrerMappingTableComponent.js +++ b/scripts/packageNTPSuperReferrerMappingTableComponent.js @@ -8,12 +8,10 @@ import path from 'path' import util from '../lib/util.js' import ntpUtil from '../lib/ntpUtil.js' -const stageFiles = (version, outputDir) => { - const files = [ - { path: getOriginalManifest(), outputName: 'manifest.json' }, - { path: path.join(path.resolve(), 'build', 'ntp-super-referrer', 'resources', 'mapping-table', 'mapping-table.json') } - ] - util.stageFiles(files, version, outputDir) +const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer', 'mapping-table') + +const getOriginalManifest = () => { + return path.join(path.resolve(), 'build', 'ntp-super-referrer', 'mapping-table-manifest.json') } const generateManifestFile = async (publicKey) => { @@ -28,26 +26,36 @@ const generateManifestFile = async (publicKey) => { fs.writeFileSync(manifestFile, JSON.stringify(manifestContent)) } -const getOriginalManifest = () => { - return path.join(path.resolve(), 'build', 'ntp-super-referrer', 'mapping-table-manifest.json') +class NTPSuperReferrerMappingTableComponent { + constructor (privateKeyFile) { + const [publicKey, componentId] = ntpUtil.generatePublicKeyAndID(privateKeyFile) + this.publicKey = publicKey + this.componentId = componentId + } + + stagingDir = path.join(rootBuildDir, 'staging') + crxFile = path.join(rootBuildDir, 'output', 'ntp-super-referrer-mapping-table.crx') + + async stageFiles (version, outputDir) { + generateManifestFile(this.publicKey) + const files = [ + { path: getOriginalManifest(), outputName: 'manifest.json' }, + { path: path.join(path.resolve(), 'build', 'ntp-super-referrer', 'resources', 'mapping-table', 'mapping-table.json') } + ] + util.stageFiles(files, version, outputDir) + } } -const generateCRXFile = async (binary, endpoint, region, componentID, privateKeyFile, +const generateCRXFile = async (binary, endpoint, region, privateKeyFile, publisherProofKey) => { - const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer', 'mapping-table') - - const stagingDir = path.join(rootBuildDir, 'staging') - const crxFile = path.join(rootBuildDir, 'output', 'ntp-super-referrer-mapping-table.crx') + const descriptor = new NTPSuperReferrerMappingTableComponent(privateKeyFile) await util.prepareNextVersionCRX( binary, publisherProofKey, endpoint, region, - componentID, - stageFiles, - stagingDir, - crxFile, + descriptor, privateKeyFile, false) } @@ -67,8 +75,6 @@ if (fs.existsSync(commander.key)) { } util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - const [publicKey, componentID] = ntpUtil.generatePublicKeyAndID(privateKeyFile) - generateManifestFile(publicKey) await generateCRXFile(commander.binary, commander.endpoint, commander.region, - componentID, privateKeyFile, commander.publisherProofKey) + privateKeyFile, commander.publisherProofKey) }) diff --git a/scripts/packagePlaylist.js b/scripts/packagePlaylist.js index d77d578d..e32ef5d5 100644 --- a/scripts/packagePlaylist.js +++ b/scripts/packagePlaylist.js @@ -8,31 +8,36 @@ import path from 'path' import util from '../lib/util.js' import ntpUtil from '../lib/ntpUtil.js' -const getOriginalManifest = () => { - return path.join(path.resolve(), 'node_modules', 'playlist-component', 'manifest.json') +const rootBuildDir = path.join(path.resolve(), 'build', 'playlist') + +class Playlist { + constructor (privateKeyFile) { + const [, componentId] = ntpUtil.generatePublicKeyAndID(privateKeyFile) + this.componentId = componentId + } + + stagingDir = path.join(rootBuildDir, 'staging') + crxFile = path.join(rootBuildDir, 'output', 'playlist.crx') + + async stageFiles (version, outputDir) { + const originalManifest = path.join(path.resolve(), 'node_modules', 'playlist-component', 'manifest.json') + util.stageDir( + path.join(path.resolve(), 'node_modules', 'playlist-component'), + originalManifest, + version, + outputDir) + } } -const stageFiles = util.stageDir.bind( - undefined, - path.join(path.resolve(), 'node_modules', 'playlist-component'), - getOriginalManifest()) - -const generateCRXFile = async (binary, endpoint, region, componentID, privateKeyFile, - publisherProofKey) => { - const rootBuildDir = path.join(path.resolve(), 'build', 'playlist') - - const stagingDir = path.join(rootBuildDir, 'staging') - const crxFile = path.join(rootBuildDir, 'output', 'playlist.crx') +const generateCRXFile = async (binary, endpoint, region, privateKeyFile, publisherProofKey) => { + const descriptor = new Playlist(privateKeyFile) await util.prepareNextVersionCRX( binary, publisherProofKey, endpoint, region, - componentID, - stageFiles, - stagingDir, - crxFile, + descriptor, privateKeyFile, false) } @@ -52,7 +57,6 @@ if (fs.existsSync(commander.key)) { } util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - const [, componentID] = ntpUtil.generatePublicKeyAndID(privateKeyFile) await generateCRXFile(commander.binary, commander.endpoint, commander.region, - componentID, privateKeyFile, commander.publisherProofKey) + privateKeyFile, commander.publisherProofKey) }) diff --git a/scripts/packageTorClient.js b/scripts/packageTorClient.js index 79038806..839a5a54 100644 --- a/scripts/packageTorClient.js +++ b/scripts/packageTorClient.js @@ -63,51 +63,60 @@ const downloadTorClient = (platform) => { return torClient } +// Does a hash comparison on a file against a given hash +const verifyChecksum = (file, hash) => { + const filecontent = fs.readFileSync(file) + const computedHash = crypto.createHash('sha512').update(filecontent).digest('hex') + console.log(`${file} has hash ${computedHash}`) + return hash === computedHash +} + const getOriginalManifest = (platform) => { return path.join('manifests', 'tor-client-updater', `tor-client-updater-${platform}-manifest.json`) } +class TorClient { + constructor (platform) { + this.platform = platform + const originalManifest = getOriginalManifest(this.platform) + const parsedManifest = util.parseManifest(originalManifest) + this.componentId = util.getIDFromBase64PublicKey(parsedManifest.key) + + this.stagingDir = path.join('build', 'tor-client-updater', this.platform) + this.crxFile = path.join('build', 'tor-client-updater', `tor-client-updater-${this.platform}.crx`) + } + + privateKeyFromDir (keyDir) { + return path.join(keyDir, `tor-client-updater-${this.platform}.pem`) + } + + async stageFiles (version, outputDir) { + const torClient = downloadTorClient(this.platform) + const files = [ + { path: getOriginalManifest(this.platform), outputName: 'manifest.json' }, + { path: torClient }, + { path: path.join('resources', 'tor', 'torrc'), outputName: 'tor-torrc' } + ] + util.stageFiles(files, version, outputDir) + } +} + const packageTorClient = async (binary, endpoint, region, platform, key, publisherProofKey) => { - const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `tor-client-updater-${platform}.pem`) - const originalManifest = getOriginalManifest(platform) - const parsedManifest = util.parseManifest(originalManifest) - const id = util.getIDFromBase64PublicKey(parsedManifest.key) - const torClient = downloadTorClient(platform) + const descriptor = new TorClient(platform) - const stagingDir = path.join('build', 'tor-client-updater', platform) - const crxFile = path.join('build', 'tor-client-updater', `tor-client-updater-${platform}.crx`) + const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : descriptor.privateKeyFromDir(key) await util.prepareNextVersionCRX( binary, publisherProofKey, endpoint, region, - id, - stageFiles.bind(undefined, platform, torClient), - stagingDir, - crxFile, + descriptor, privateKeyFile, false) } -const stageFiles = (platform, torClient, version, outputDir) => { - const files = [ - { path: getOriginalManifest(platform), outputName: 'manifest.json' }, - { path: torClient }, - { path: path.join('resources', 'tor', 'torrc'), outputName: 'tor-torrc' } - ] - util.stageFiles(files, version, outputDir) -} - -// Does a hash comparison on a file against a given hash -const verifyChecksum = (file, hash) => { - const filecontent = fs.readFileSync(file) - const computedHash = crypto.createHash('sha512').update(filecontent).digest('hex') - console.log(`${file} has hash ${computedHash}`) - return hash === computedHash -} - util.installErrorHandlers() util.addCommonScriptOptions( diff --git a/scripts/packageTorPluggableTransports.js b/scripts/packageTorPluggableTransports.js index 3a198e85..9750630d 100644 --- a/scripts/packageTorPluggableTransports.js +++ b/scripts/packageTorPluggableTransports.js @@ -45,40 +45,50 @@ const getOriginalManifest = (platform) => { return path.join('manifests', TOR_PLUGGABLE_TRANSPORTS_UPDATER, `${TOR_PLUGGABLE_TRANSPORTS_UPDATER}-${platform}-manifest.json`) } -const packageTorPluggableTransports = async (binary, endpoint, region, platform, key, publisherProofKey) => { - const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `${TOR_PLUGGABLE_TRANSPORTS_UPDATER}-${platform}.pem`) - const originalManifest = getOriginalManifest(platform) - const parsedManifest = util.parseManifest(originalManifest) - const id = util.getIDFromBase64PublicKey(parsedManifest.key) +class TorPluggableTransports { + constructor (platform) { + this.platform = platform + + const originalManifest = getOriginalManifest(this.platform) + const parsedManifest = util.parseManifest(originalManifest) + this.componentId = util.getIDFromBase64PublicKey(parsedManifest.key) - const snowflake = downloadTorPluggableTransport(platform, 'snowflake') - const obfs4 = downloadTorPluggableTransport(platform, 'obfs4') + this.stagingDir = path.join('build', TOR_PLUGGABLE_TRANSPORTS_UPDATER, this.platform) + this.crxFile = path.join('build', TOR_PLUGGABLE_TRANSPORTS_UPDATER, `${TOR_PLUGGABLE_TRANSPORTS_UPDATER}-${this.platform}.crx`) + } - const stagingDir = path.join('build', TOR_PLUGGABLE_TRANSPORTS_UPDATER, platform) - const crxFile = path.join('build', TOR_PLUGGABLE_TRANSPORTS_UPDATER, `${TOR_PLUGGABLE_TRANSPORTS_UPDATER}-${platform}.crx`) + privateKeyFromDir (keyDir) { + return path.join(keyDir, `${TOR_PLUGGABLE_TRANSPORTS_UPDATER}-${this.platform}.pem`) + } + + async stageFiles (version, outputDir) { + const snowflake = downloadTorPluggableTransport(this.platform, 'snowflake') + const obfs4 = downloadTorPluggableTransport(this.platform, 'obfs4') + + const files = [ + { path: getOriginalManifest(this.platform), outputName: 'manifest.json' }, + { path: snowflake }, + { path: obfs4 } + ] + util.stageFiles(files, version, outputDir) + } +} + +const packageTorPluggableTransports = async (binary, endpoint, region, platform, key, publisherProofKey) => { + const descriptor = new TorPluggableTransports(platform) + + const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : descriptor.privateKeyFromDir(key) await util.prepareNextVersionCRX( binary, publisherProofKey, endpoint, region, - id, - stageFiles.bind(undefined, platform, snowflake, obfs4), - stagingDir, - crxFile, + descriptor, privateKeyFile, false) } -const stageFiles = (platform, snowflake, obfs4, version, outputDir) => { - const files = [ - { path: getOriginalManifest(platform), outputName: 'manifest.json' }, - { path: snowflake }, - { path: obfs4 } - ] - util.stageFiles(files, version, outputDir) -} - util.installErrorHandlers() util.addCommonScriptOptions( From f867caed639bd018d9cc1e7994fd5c9efa7df378 Mon Sep 17 00:00:00 2001 From: Anton Lazarev Date: Wed, 7 Feb 2024 14:45:06 -0800 Subject: [PATCH 4/5] split packageEthereumRemoteClient and packageWalletDataFiles --- package.json | 4 +- ...nent.js => packageEthereumRemoteClient.js} | 45 ++------- scripts/packageWalletDataFiles.js | 92 +++++++++++++++++++ 3 files changed, 100 insertions(+), 41 deletions(-) rename scripts/{packageComponent.js => packageEthereumRemoteClient.js} (60%) create mode 100644 scripts/packageWalletDataFiles.js diff --git a/package.json b/package.json index 72fc3e5f..157dceb0 100644 --- a/package.json +++ b/package.json @@ -39,12 +39,12 @@ "generate-user-model-installer-updates": "node scripts/generateBraveAdsResourcesComponentInputFiles.js", "lint": "standard lib scripts test", "package-brave-player": "node ./scripts/packageBravePlayer", - "package-ethereum-remote-client": "node ./scripts/packageComponent --type ethereum-remote-client", + "package-ethereum-remote-client": "node ./scripts/packageEthereumRemoteClient", "package-ad-block": "node ./scripts/packageAdBlock", "package-tor-client": "node ./scripts/packageTorClient", "package-tor-pluggable-transports": "node ./scripts/packageTorPluggableTransports", "package-ipfs-daemon": "node ./scripts/packageIpfsDaemon", - "package-wallet-data-files": "node ./scripts/packageComponent --type wallet-data-files-updater", + "package-wallet-data-files": "node ./scripts/packageWalletDataFiles", "package-local-data-files": "node ./scripts/packageLocalDataFiles", "package-ntp-background-images": "node ./scripts/packageNTPBackgroundImagesComponent", "package-ntp-sponsored-images": "node ./scripts/ntp-sponsored-images/package", diff --git a/scripts/packageComponent.js b/scripts/packageEthereumRemoteClient.js similarity index 60% rename from scripts/packageComponent.js rename to scripts/packageEthereumRemoteClient.js index a261f287..5df52c4b 100644 --- a/scripts/packageComponent.js +++ b/scripts/packageEthereumRemoteClient.js @@ -21,14 +21,13 @@ class EthereumRemoteClient { this.componentId = util.getIDFromBase64PublicKey(parsedManifest.key) } - componentType = 'ethereum-remote-client' packageDir = path.join('node_modules', 'ethereum-remote-client') - stagingDir = path.join('build', this.componentType) - crxFile = path.join(this.stagingDir, `${this.componentType}.crx`) + stagingDir = path.join('build', 'ethereum-remote-client') + crxFile = path.join(this.stagingDir, 'ethereum-remote-client.crx') privateKeyFromDir (keyDir) { - return path.join(keyDir, `${this.componentType}.pem`) + return path.join(keyDir, 'ethereum-remote-client.pem') } async stageFiles (version, outputDir) { @@ -36,39 +35,9 @@ class EthereumRemoteClient { } } -class WalletDataFilesUpdater { - constructor () { - const originalManifest = getOriginalManifest(this.packageDir) - const parsedManifest = util.parseManifest(originalManifest) - this.componentId = util.getIDFromBase64PublicKey(parsedManifest.key) - } - - componentType = 'wallet-data-files-updater' - packageDir = path.join('node_modules', 'brave-wallet-lists') - - stagingDir = path.join('build', this.componentType) - crxFile = path.join(this.stagingDir, `${this.componentType}.crx`) - - privateKeyFromDir (keyDir) { - return path.join(keyDir, `${this.componentType}.pem`) - } - - async stageFiles (version, outputDir) { - util.stageDir(this.packageDir, getOriginalManifest(this.packageDir), version, outputDir) - fs.unlinkSync(path.join(outputDir, 'package.json')) - } -} - -const generateCRXFile = async (binary, endpoint, region, componentType, key, +const generateCRXFile = async (binary, endpoint, region, key, publisherProofKey, localRun) => { - let descriptor - if (componentType === 'ethereum-remote-client') { - descriptor = new EthereumRemoteClient() - } else if (componentType === 'wallet-data-files-updater') { - descriptor = new WalletDataFilesUpdater() - } else { - throw new Error('Unrecognized component extension type: ' + commander.type) - } + const descriptor = new EthereumRemoteClient() let privateKeyFile = '' if (!localRun) { @@ -87,8 +56,7 @@ const generateCRXFile = async (binary, endpoint, region, componentType, key, const processJob = async (commander, keyParam) => { await generateCRXFile(commander.binary, commander.endpoint, - commander.region, commander.type, keyParam, - commander.publisherProofKey, + commander.region, keyParam, commander.publisherProofKey, commander.localRun) } @@ -98,7 +66,6 @@ util.addCommonScriptOptions( commander .option('-d, --keys-directory ', 'directory containing private keys for signing crx files') .option('-f, --key-file ', 'private key file for signing crx', 'key.pem') - .option('-t, --type ', 'component extension type', /^(ethereum-remote-client|wallet-data-files-updater)$/i) .option('-l, --local-run', 'Runs updater job without connecting anywhere remotely')) .parse(process.argv) diff --git a/scripts/packageWalletDataFiles.js b/scripts/packageWalletDataFiles.js new file mode 100644 index 00000000..d834f551 --- /dev/null +++ b/scripts/packageWalletDataFiles.js @@ -0,0 +1,92 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Example usage: +// npm run package-wallet-data-files -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --key-file path/to/wallet-data-files-updater.pem + +import commander from 'commander' +import fs from 'fs-extra' +import path from 'path' +import util from '../lib/util.js' + +const getOriginalManifest = (packageDir) => { + return path.join(packageDir, 'manifest.json') +} + +class WalletDataFilesUpdater { + constructor () { + const originalManifest = getOriginalManifest(this.packageDir) + const parsedManifest = util.parseManifest(originalManifest) + this.componentId = util.getIDFromBase64PublicKey(parsedManifest.key) + } + + packageDir = path.join('node_modules', 'brave-wallet-lists') + + stagingDir = path.join('build', 'wallet-data-files-updater') + crxFile = path.join(this.stagingDir, 'wallet-data-files-updater.crx') + + privateKeyFromDir (keyDir) { + return path.join(keyDir, 'wallet-data-files-updater.pem') + } + + async stageFiles (version, outputDir) { + util.stageDir(this.packageDir, getOriginalManifest(this.packageDir), version, outputDir) + fs.unlinkSync(path.join(outputDir, 'package.json')) + } +} + +const generateCRXFile = async (binary, endpoint, region, key, + publisherProofKey, localRun) => { + const descriptor = new WalletDataFilesUpdater() + + let privateKeyFile = '' + if (!localRun) { + privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : descriptor.privateKeyFromDir(key) + } + + await util.prepareNextVersionCRX( + binary, + publisherProofKey, + endpoint, + region, + descriptor, + privateKeyFile, + localRun) +} + +const processJob = async (commander, keyParam) => { + await generateCRXFile(commander.binary, commander.endpoint, + commander.region, keyParam, + commander.publisherProofKey, + commander.localRun) +} + +util.installErrorHandlers() + +util.addCommonScriptOptions( + commander + .option('-d, --keys-directory ', 'directory containing private keys for signing crx files') + .option('-f, --key-file ', 'private key file for signing crx', 'key.pem') + .option('-l, --local-run', 'Runs updater job without connecting anywhere remotely')) + .parse(process.argv) + +let keyParam = '' + +if (!commander.localRun) { + if (fs.existsSync(commander.keyFile)) { + keyParam = commander.keyFile + } else if (fs.existsSync(commander.keysDirectory)) { + keyParam = commander.keysDirectory + } else { + throw new Error('Missing or invalid private key file/directory') + } +} + +if (!commander.localRun) { + util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { + await processJob(commander, keyParam) + }) +} else { + processJob(commander, keyParam) +} From 263322d4eba1b87db867d514b5cfa02bff0e1b16 Mon Sep 17 00:00:00 2001 From: Anton Lazarev Date: Thu, 8 Feb 2024 15:28:19 -0800 Subject: [PATCH 5/5] move common packaging logic into shared packageComponent script --- lib/util.js | 13 ---- scripts/ntp-sponsored-images/package.js | 49 +++----------- scripts/packageAdBlock.js | 52 ++------------- scripts/packageBraveAdsResourcesComponent.js | 49 +++----------- scripts/packageBravePlayer.js | 56 +--------------- scripts/packageComponent.js | 58 +++++++++++++++++ scripts/packageEthereumRemoteClient.js | 58 +---------------- scripts/packageIpfsDaemon.js | 64 ++++--------------- scripts/packageLocalDataFiles.js | 56 +--------------- .../packageNTPBackgroundImagesComponent.js | 37 ++--------- scripts/packageNTPSuperReferrerComponent.js | 40 ++---------- ...geNTPSuperReferrerMappingTableComponent.js | 37 ++--------- scripts/packagePlaylist.js | 37 ++--------- scripts/packageTorClient.js | 48 ++------------ scripts/packageTorPluggableTransports.js | 45 ++----------- scripts/packageWalletDataFiles.js | 58 +---------------- 16 files changed, 137 insertions(+), 620 deletions(-) create mode 100644 scripts/packageComponent.js diff --git a/lib/util.js b/lib/util.js index 239550d2..c3ca3ed1 100644 --- a/lib/util.js +++ b/lib/util.js @@ -598,18 +598,6 @@ const updateDynamoDB = (endpoint, region, id, version, hash, name, disabled, con return dynamodb.send(new PutItemCommand(params)) } -const addCommonScriptOptions = (command) => { - return command - .option('-b, --binary ', - 'Path to the Chromium based executable to use to generate the CRX file') - .option('-p, --publisher-proof-key ', - 'File containing private key for generating publisher proof') - - // If setup locally, use --endpoint http://localhost:8000 - .option('-e, --endpoint ', 'DynamoDB endpoint to connect to', '') - .option('-r, --region ', 'The AWS region to use', 'us-west-2') -} - const escapeStringForJSON = str => { if (typeof str !== 'string') { throw new Error('Not a string: ' + JSON.stringify(str)) @@ -667,7 +655,6 @@ export default { parseManifest, uploadCRXFile, updateDBForCRXFile, - addCommonScriptOptions, escapeStringForJSON, copyManifestWithVersion, stageDir, diff --git a/scripts/ntp-sponsored-images/package.js b/scripts/ntp-sponsored-images/package.js index 16a7aaa9..eaa7a2e0 100644 --- a/scripts/ntp-sponsored-images/package.js +++ b/scripts/ntp-sponsored-images/package.js @@ -3,12 +3,12 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // you can obtain one at http://mozilla.org/MPL/2.0/. -import commander from 'commander' import fs from 'fs-extra' import { mkdirp } from 'mkdirp' import path from 'path' import util from '../../lib/util.js' import params from './params.js' +import { getPackagingArgs, packageComponent } from './packageComponent' const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-sponsored-images') @@ -66,45 +66,14 @@ class NtpSponsoredImages { } } -const generateCRXFile = async (binary, endpoint, region, keyDir, platformRegion, - componentData, publisherProofKey) => { - const descriptor = new NtpSponsoredImages(platformRegion, componentData) +const args = getPackagingArgs([ + ['-t, --target-regions ', 'Comma separated list of regions that should generate SI component. For example: "AU-android,US-desktop,GB-ios"', ''], + ['-u, --excluded-target-regions ', 'Comma separated list of regions that should not generate SI component. For example: "AU-android,US-desktop,GB-ios"', ''] +]) - const privateKeyFile = descriptor.privateKeyFromDir(keyDir) +const targetComponents = params.getTargetComponents(args.targetRegions, args.excludedTargetRegions) - await util.prepareNextVersionCRX( - binary, - publisherProofKey, - endpoint, - region, - descriptor, - privateKeyFile, - false) +for (const platformRegion of Object.keys(targetComponents)) { + const componentData = targetComponents[platformRegion] + packageComponent(args, new NtpSponsoredImages(platformRegion, componentData)) } - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files') - .option('-t, --target-regions ', 'Comma separated list of regions that should generate SI component. For example: "AU-android,US-desktop,GB-ios"', '') - .option('-u, --excluded-target-regions ', 'Comma separated list of regions that should not generate SI component. For example: "AU-android,US-desktop,GB-ios"', '')) - .parse(process.argv) - -let keyDir = '' -if (fs.existsSync(commander.keysDirectory)) { - keyDir = commander.keysDirectory -} else { - throw new Error('Missing or invalid private key directory') -} - -const targetComponents = params.getTargetComponents(commander.targetRegions, commander.excludedTargetRegions) - -util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - for (const platformRegion of Object.keys(targetComponents)) { - const componentData = targetComponents[platformRegion] - await generateCRXFile(commander.binary, commander.endpoint, commander.region, - keyDir, platformRegion, componentData, - commander.publisherProofKey) - } -}) diff --git a/scripts/packageAdBlock.js b/scripts/packageAdBlock.js index 3c8beefb..7f1dfdbd 100644 --- a/scripts/packageAdBlock.js +++ b/scripts/packageAdBlock.js @@ -6,11 +6,11 @@ // Example usage: // npm run package-ad-block -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --key-file path/to/ad-block-updater-regional-component-keys -import commander from 'commander' import fs from 'fs-extra' import path from 'path' import Sentry from '../lib/sentry.js' import util from '../lib/util.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' import { downloadListsForEntry, generatePlaintextListFromLists, @@ -169,23 +169,6 @@ class AdblockResources { } } -const processComponent = async (binary, endpoint, region, keyParam, - publisherProofKey, localRun, descriptor) => { - let privateKeyFile = '' - if (!localRun) { - privateKeyFile = !fs.lstatSync(keyParam).isDirectory() ? keyParam : descriptor.privateKeyFromDir(keyParam) - } - - await util.prepareNextVersionCRX( - binary, - publisherProofKey, - endpoint, - region, - descriptor, - privateKeyFile, - localRun) -} - const getAdblockComponentDescriptors = async () => { const output = [ new AdblockCatalog(), @@ -195,32 +178,7 @@ const getAdblockComponentDescriptors = async () => { return output } -const processJob = async (commander, keyDir) => { - (await getAdblockComponentDescriptors()) - .map(processComponent.bind(null, commander.binary, commander.endpoint, - commander.region, keyDir, - commander.publisherProofKey, - commander.localRun)) -} - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files') - .option('-l, --local-run', 'Runs updater job without connecting anywhere remotely')) - .parse(process.argv) - -if (!commander.localRun) { - let keyDir = '' - if (fs.existsSync(commander.keysDirectory)) { - keyDir = commander.keysDirectory - } else { - throw new Error('Missing or invalid private key file/directory') - } - util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - await processJob(commander, keyDir) - }) -} else { - processJob(commander, undefined) -} +const args = getPackagingArgs() +await Promise.all((await getAdblockComponentDescriptors()).map(descriptor => + packageComponent(args, descriptor) +)) diff --git a/scripts/packageBraveAdsResourcesComponent.js b/scripts/packageBraveAdsResourcesComponent.js index 45113a12..89fea4a0 100644 --- a/scripts/packageBraveAdsResourcesComponent.js +++ b/scripts/packageBraveAdsResourcesComponent.js @@ -2,11 +2,11 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ -import commander from 'commander' import fs from 'fs-extra' import { mkdirp } from 'mkdirp' import path from 'path' import util from '../lib/util.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' const getComponentDataList = () => { return [ @@ -245,11 +245,11 @@ const getOriginalManifest = (locale) => { return path.join(getManifestsDir(), `${locale}-manifest.json`) } -const generateManifestFile = (componentData) => { - const manifestFile = getOriginalManifest(componentData.locale) +const generateManifestFile = (locale, key) => { + const manifestFile = getOriginalManifest(locale) const manifestContent = { description: 'Brave Ads Resources Component', - key: componentData.key, + key, manifest_version: 2, name: 'Brave Ads Resources', version: '0.0.0' @@ -271,7 +271,7 @@ class BraveAdsResourcesComponent { } async stageFiles (version, outputDir) { - generateManifestFile(this.componentData) + generateManifestFile(this.locale, this.key) util.stageDir( path.join(path.resolve(), 'build', 'user-model-installer', 'resources', this.locale, '/'), getOriginalManifest(this.locale), @@ -280,39 +280,8 @@ class BraveAdsResourcesComponent { } } -const generateCRXFile = async (binary, endpoint, region, keyDir, publisherProofKey, - componentData) => { - const descriptor = new BraveAdsResourcesComponent(componentData) +const args = getPackagingArgs() - const privateKeyFile = descriptor.privateKeyFromDir(keyDir) - - await util.prepareNextVersionCRX( - binary, - publisherProofKey, - endpoint, - region, - descriptor, - privateKeyFile, - false) -} - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files')) - .parse(process.argv) - -let keyDir = '' -if (fs.existsSync(commander.keysDirectory)) { - keyDir = commander.keysDirectory -} else { - throw new Error('Missing or invalid private key directory') -} - -util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - await Promise.all(getComponentDataList().map( - generateCRXFile.bind(null, commander.binary, commander.endpoint, - commander.region, keyDir, - commander.publisherProofKey))) -}) +await Promise.all(getComponentDataList().map(componentData => + packageComponent(args, new BraveAdsResourcesComponent(componentData)) +)) diff --git a/scripts/packageBravePlayer.js b/scripts/packageBravePlayer.js index 66d5d822..1a5d7441 100644 --- a/scripts/packageBravePlayer.js +++ b/scripts/packageBravePlayer.js @@ -10,10 +10,9 @@ // Example usage: // npm run package-brave-player -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --key-file path/to/brave-player.pem -import commander from 'commander' -import fs from 'fs-extra' import path from 'path' import util from '../lib/util.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' const getOriginalManifest = () => { return path.join('component-data', 'brave-player', 'manifest.json') @@ -38,54 +37,5 @@ class BravePlayer { } } -const packageBravePlayer = async (binary, endpoint, region, key, publisherProofKey, localRun) => { - const descriptor = new BravePlayer() - - let privateKeyFile = '' - if (!localRun) { - privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : descriptor.privateKeyFromDir(key) - } - - await util.prepareNextVersionCRX( - binary, - publisherProofKey, - endpoint, - region, - descriptor, - privateKeyFile, - localRun) -} - -const processJob = async (commander, keyParam) => { - await packageBravePlayer(commander.binary, commander.endpoint, commander.region, - keyParam, commander.publisherProofKey, commander.localRun) -} - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files') - .option('-f, --key-file ', 'private key file for signing crx', 'key.pem') - .option('-l, --local-run', 'Runs updater job without connecting anywhere remotely')) - .parse(process.argv) - -let keyParam = '' - -if (!commander.localRun) { - if (fs.existsSync(commander.keyFile)) { - keyParam = commander.keyFile - } else if (fs.existsSync(commander.keysDirectory)) { - keyParam = commander.keysDirectory - } else { - throw new Error('Missing or invalid private key file/directory') - } -} - -if (!commander.localRun) { - util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - await processJob(commander, keyParam) - }) -} else { - processJob(commander, keyParam) -} +const args = getPackagingArgs() +packageComponent(args, new BravePlayer()) diff --git a/scripts/packageComponent.js b/scripts/packageComponent.js new file mode 100644 index 00000000..2b9c4107 --- /dev/null +++ b/scripts/packageComponent.js @@ -0,0 +1,58 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import commander from 'commander' +import fs from 'fs/promises' +import util from '../lib/util.js' + +const getPackagingArgs = (additionalParams) => { + util.installErrorHandlers() + + let setup = commander + .option('-d, --keys-directory ', 'directory containing private keys for signing crx files') + .option('-f, --key-file ', 'private key file for signing crx') + .option('-l, --local-run', 'Runs updater job without connecting anywhere remotely') + .option('-b, --binary ', 'Path to the Chromium based executable to use to generate the CRX file') + .option('-p, --publisher-proof-key ', 'File containing private key for generating publisher proof') + .option('-e, --endpoint ', 'DynamoDB endpoint to connect to', '') + .option('-r, --region ', 'The AWS region to use', 'us-west-2') + + if (additionalParams !== undefined) { + for (const param of additionalParams) { + setup = setup.option(...param) + } + } + + return setup.parse(process.argv) +} + +const packageComponent = async (packagingArgs, componentClass) => { + let privateKeyFile = '' + + if (packagingArgs.keyFile !== undefined && (await fs.lstat(packagingArgs.keyFile)).isFile()) { + privateKeyFile = packagingArgs.keyFile + } else if (packagingArgs.keysDirectory !== undefined && (await fs.lstat(packagingArgs.keysDirectory)).isDirectory()) { + privateKeyFile = componentClass.privateKeyFromDir(packagingArgs.keysDirectory) + } else if (packagingArgs.localRun !== true) { + throw new Error('Missing or invalid private key file/directory') + } + + if (packagingArgs.localRun !== true) { + await util.createTableIfNotExists(packagingArgs.endpoint, packagingArgs.region) + } + + await util.prepareNextVersionCRX( + packagingArgs.binary, + packagingArgs.publisherProofKey, + packagingArgs.endpoint, + packagingArgs.region, + componentClass, + privateKeyFile, + packagingArgs.localRun) +} + +export { + getPackagingArgs, + packageComponent +} diff --git a/scripts/packageEthereumRemoteClient.js b/scripts/packageEthereumRemoteClient.js index 5df52c4b..7d076875 100644 --- a/scripts/packageEthereumRemoteClient.js +++ b/scripts/packageEthereumRemoteClient.js @@ -5,10 +5,9 @@ // Example usage: // npm run package-ethereum-remote-client -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --key-file path/to/ethereum-remote-client.pem -import commander from 'commander' -import fs from 'fs-extra' import path from 'path' import util from '../lib/util.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' const getOriginalManifest = (packageDir) => { return path.join(packageDir, 'manifest.json') @@ -35,56 +34,5 @@ class EthereumRemoteClient { } } -const generateCRXFile = async (binary, endpoint, region, key, - publisherProofKey, localRun) => { - const descriptor = new EthereumRemoteClient() - - let privateKeyFile = '' - if (!localRun) { - privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : descriptor.privateKeyFromDir(key) - } - - await util.prepareNextVersionCRX( - binary, - publisherProofKey, - endpoint, - region, - descriptor, - privateKeyFile, - localRun) -} - -const processJob = async (commander, keyParam) => { - await generateCRXFile(commander.binary, commander.endpoint, - commander.region, keyParam, commander.publisherProofKey, - commander.localRun) -} - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files') - .option('-f, --key-file ', 'private key file for signing crx', 'key.pem') - .option('-l, --local-run', 'Runs updater job without connecting anywhere remotely')) - .parse(process.argv) - -let keyParam = '' - -if (!commander.localRun) { - if (fs.existsSync(commander.keyFile)) { - keyParam = commander.keyFile - } else if (fs.existsSync(commander.keysDirectory)) { - keyParam = commander.keysDirectory - } else { - throw new Error('Missing or invalid private key file/directory') - } -} - -if (!commander.localRun) { - util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - await processJob(commander, keyParam) - }) -} else { - processJob(commander, keyParam) -} +const args = getPackagingArgs() +packageComponent(args, new EthereumRemoteClient()) diff --git a/scripts/packageIpfsDaemon.js b/scripts/packageIpfsDaemon.js index ba7c8a26..6a7da056 100644 --- a/scripts/packageIpfsDaemon.js +++ b/scripts/packageIpfsDaemon.js @@ -5,10 +5,9 @@ // Example usage: // npm run package-ipfs-daemon -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --keys-directory path/to/key/dir -import commander from 'commander' -import fs from 'fs' import path from 'path' import util from '../lib/util.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' const ipfsVersion = '0.24.0' const getIpfsDaemonPath = (os, arch) => { @@ -50,51 +49,16 @@ class IpfsDaemon { } } -const packageIpfsDaemon = async (binary, endpoint, region, os, arch, key, - publisherProofKey) => { - const descriptor = new IpfsDaemon(os, arch) - - const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : descriptor.privateKeyFromDir(key) - - await util.prepareNextVersionCRX( - binary, - publisherProofKey, - endpoint, - region, - descriptor, - privateKeyFile, - false) -} - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files', 'abc') - .option('-f, --key-file ', 'private key file for signing crx', 'key.pem')) - .parse(process.argv) - -let keyParam = '' - -if (fs.existsSync(commander.keyFile)) { - keyParam = commander.keyFile -} else if (fs.existsSync(commander.keysDirectory)) { - keyParam = commander.keysDirectory -} else { - throw new Error('Missing or invalid private key file/directory') -} - -util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - await packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, - 'darwin', 'amd64', keyParam, commander.publisherProofKey) - await packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, - 'darwin', 'arm64', keyParam, commander.publisherProofKey) - await packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, - 'linux', 'amd64', keyParam, commander.publisherProofKey) - await packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, - 'linux', 'arm64', keyParam, commander.publisherProofKey) - await packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, - 'win32', 'amd64', keyParam, commander.publisherProofKey) - await packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, - 'win32', 'arm64', keyParam, commander.publisherProofKey) -}) +const osArchVariants = [ + ['darwin', 'amd64'], + ['darwin', 'arm64'], + ['linux', 'amd64'], + ['linux', 'arm64'], + ['win32', 'amd64'], + ['win32', 'arm64'] +] + +const args = getPackagingArgs() +Promise.all(osArchVariants.map(([os, arch]) => + packageComponent(args, new IpfsDaemon(os, arch)) +)) diff --git a/scripts/packageLocalDataFiles.js b/scripts/packageLocalDataFiles.js index 9e8ddf76..ea864609 100644 --- a/scripts/packageLocalDataFiles.js +++ b/scripts/packageLocalDataFiles.js @@ -5,12 +5,11 @@ // Example usage: // npm run package-local-data-files -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --key-file path/to/local-data-files-updater.pem -import commander from 'commander' -import fs from 'fs-extra' import { mkdirp } from 'mkdirp' import path from 'path' import recursive from 'recursive-readdir-sync' import util from '../lib/util.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' const componentType = 'local-data-files-updater' const datFileName = 'default' @@ -58,54 +57,5 @@ class LocalDataFiles { } } -const generateCRXFile = async (binary, endpoint, region, key, publisherProofKey, localRun) => { - const descriptor = new LocalDataFiles() - - let privateKeyFile = '' - if (!localRun) { - privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : descriptor.privateKeyFromDir(key) - } - - await util.prepareNextVersionCRX( - binary, - publisherProofKey, - endpoint, - region, - descriptor, - privateKeyFile, - localRun) -} - -const processJob = async (commander, keyParam) => { - await generateCRXFile(commander.binary, commander.endpoint, commander.region, - keyParam, commander.publisherProofKey, commander.localRun) -} - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files') - .option('-f, --key-file ', 'private key file for signing crx', 'key.pem') - .option('-l, --local-run', 'Runs updater job without connecting anywhere remotely')) - .parse(process.argv) - -let keyParam = '' - -if (!commander.localRun) { - if (fs.existsSync(commander.keyFile)) { - keyParam = commander.keyFile - } else if (fs.existsSync(commander.keysDirectory)) { - keyParam = commander.keysDirectory - } else { - throw new Error('Missing or invalid private key file/directory') - } -} - -if (!commander.localRun) { - util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - await processJob(commander, keyParam) - }) -} else { - processJob(commander, keyParam) -} +const args = getPackagingArgs() +packageComponent(args, new LocalDataFiles()) diff --git a/scripts/packageNTPBackgroundImagesComponent.js b/scripts/packageNTPBackgroundImagesComponent.js index 1ae7b63e..c30f8bc6 100644 --- a/scripts/packageNTPBackgroundImagesComponent.js +++ b/scripts/packageNTPBackgroundImagesComponent.js @@ -2,11 +2,11 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ -import commander from 'commander' import fs from 'fs-extra' import path from 'path' import util from '../lib/util.js' import ntpUtil from '../lib/ntpUtil.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-background-images') @@ -46,35 +46,8 @@ class NTPBackgroundImagesComponent { } } -const generateCRXFile = async (binary, endpoint, region, privateKeyFile, - publisherProofKey) => { - const descriptor = new NTPBackgroundImagesComponent(privateKeyFile) - - await util.prepareNextVersionCRX( - binary, - publisherProofKey, - endpoint, - region, - descriptor, - privateKeyFile, - false) -} - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-k, --key ', 'file containing private key for signing crx file')) - .parse(process.argv) - -let privateKeyFile = '' -if (fs.existsSync(commander.key)) { - privateKeyFile = commander.key -} else { - throw new Error('Missing or invalid private key') +const args = getPackagingArgs() +if (args.keyFile === undefined) { + throw new Error('--key-file is required') } - -util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - await generateCRXFile(commander.binary, commander.endpoint, commander.region, - privateKeyFile, commander.publisherProofKey) -}) +packageComponent(args, new NTPBackgroundImagesComponent(args.keyFile)) diff --git a/scripts/packageNTPSuperReferrerComponent.js b/scripts/packageNTPSuperReferrerComponent.js index f868cd10..c9f14ef9 100644 --- a/scripts/packageNTPSuperReferrerComponent.js +++ b/scripts/packageNTPSuperReferrerComponent.js @@ -2,11 +2,11 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ -import commander from 'commander' import fs from 'fs-extra' import path from 'path' import util from '../lib/util.js' import ntpUtil from '../lib/ntpUtil.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer') @@ -46,36 +46,10 @@ class NTPSuperReferrerComponent { } } -const generateCRXFile = async (binary, endpoint, region, superReferrerName, - privateKeyFile, publisherProofKey) => { - const descriptor = new NTPSuperReferrerComponent(superReferrerName, privateKeyFile) - - await util.prepareNextVersionCRX( - binary, - publisherProofKey, - endpoint, - region, - descriptor, - privateKeyFile, - false) -} - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-n, --super-referrer-name ', 'super referrer name for this component') - .option('-k, --key ', 'file containing private key for signing crx file')) - .parse(process.argv) - -let privateKeyFile = '' -if (fs.existsSync(commander.key)) { - privateKeyFile = commander.key -} else { - throw new Error('Missing or invalid private key') +const args = getPackagingArgs([ + ['-n, --super-referrer-name ', 'super referrer name for this component'] +]) +if (args.keyFile === undefined) { + throw new Error('--key-file is required') } - -util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - await generateCRXFile(commander.binary, commander.endpoint, commander.region, - commander.superReferrerName, privateKeyFile, commander.publisherProofKey) -}) +packageComponent(args, new NTPSuperReferrerComponent(args.superReferrerName, args.keyFile)) diff --git a/scripts/packageNTPSuperReferrerMappingTableComponent.js b/scripts/packageNTPSuperReferrerMappingTableComponent.js index 014a9cbc..ac2d57a8 100644 --- a/scripts/packageNTPSuperReferrerMappingTableComponent.js +++ b/scripts/packageNTPSuperReferrerMappingTableComponent.js @@ -2,11 +2,11 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ -import commander from 'commander' import fs from 'fs-extra' import path from 'path' import util from '../lib/util.js' import ntpUtil from '../lib/ntpUtil.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer', 'mapping-table') @@ -46,35 +46,8 @@ class NTPSuperReferrerMappingTableComponent { } } -const generateCRXFile = async (binary, endpoint, region, privateKeyFile, - publisherProofKey) => { - const descriptor = new NTPSuperReferrerMappingTableComponent(privateKeyFile) - - await util.prepareNextVersionCRX( - binary, - publisherProofKey, - endpoint, - region, - descriptor, - privateKeyFile, - false) -} - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-k, --key ', 'file containing private key for signing crx file')) - .parse(process.argv) - -let privateKeyFile = '' -if (fs.existsSync(commander.key)) { - privateKeyFile = commander.key -} else { - throw new Error('Missing or invalid private key') +const args = getPackagingArgs() +if (args.keyFile === undefined) { + throw new Error('--key-file is required') } - -util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - await generateCRXFile(commander.binary, commander.endpoint, commander.region, - privateKeyFile, commander.publisherProofKey) -}) +packageComponent(args, new NTPSuperReferrerMappingTableComponent(args.keyFile)) diff --git a/scripts/packagePlaylist.js b/scripts/packagePlaylist.js index e32ef5d5..c8856e11 100644 --- a/scripts/packagePlaylist.js +++ b/scripts/packagePlaylist.js @@ -2,11 +2,10 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ -import commander from 'commander' -import fs from 'fs-extra' import path from 'path' import util from '../lib/util.js' import ntpUtil from '../lib/ntpUtil.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' const rootBuildDir = path.join(path.resolve(), 'build', 'playlist') @@ -29,34 +28,8 @@ class Playlist { } } -const generateCRXFile = async (binary, endpoint, region, privateKeyFile, publisherProofKey) => { - const descriptor = new Playlist(privateKeyFile) - - await util.prepareNextVersionCRX( - binary, - publisherProofKey, - endpoint, - region, - descriptor, - privateKeyFile, - false) -} - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-k, --key ', 'file containing private key for signing crx file')) - .parse(process.argv) - -let privateKeyFile = '' -if (fs.existsSync(commander.key)) { - privateKeyFile = commander.key -} else { - throw new Error('Missing or invalid private key') +const args = getPackagingArgs() +if (args.keyFile === undefined) { + throw new Error('--key-file is required') } - -util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - await generateCRXFile(commander.binary, commander.endpoint, commander.region, - privateKeyFile, commander.publisherProofKey) -}) +packageComponent(args, new Playlist(args.keyFile)) diff --git a/scripts/packageTorClient.js b/scripts/packageTorClient.js index 839a5a54..cb1b28aa 100644 --- a/scripts/packageTorClient.js +++ b/scripts/packageTorClient.js @@ -5,13 +5,13 @@ // Example usage: // npm run package-tor-client -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --keys-directory path/to/key/dir -import commander from 'commander' import crypto from 'crypto' import { execSync } from 'child_process' import fs from 'fs' import { mkdirp } from 'mkdirp' import path from 'path' import util from '../lib/util.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' // Downloads the current (platform-specific) Tor client from S3 const downloadTorClient = (platform) => { @@ -101,47 +101,7 @@ class TorClient { } } -const packageTorClient = async (binary, endpoint, region, platform, key, - publisherProofKey) => { - const descriptor = new TorClient(platform) +const platforms = ['darwin', 'linux', 'linux-arm64', 'win32'] - const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : descriptor.privateKeyFromDir(key) - - await util.prepareNextVersionCRX( - binary, - publisherProofKey, - endpoint, - region, - descriptor, - privateKeyFile, - false) -} - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files', 'abc') - .option('-f, --key-file ', 'private key file for signing crx', 'key.pem')) - .parse(process.argv) - -let keyParam = '' - -if (fs.existsSync(commander.keyFile)) { - keyParam = commander.keyFile -} else if (fs.existsSync(commander.keysDirectory)) { - keyParam = commander.keysDirectory -} else { - throw new Error('Missing or invalid private key file/directory') -} - -util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - await packageTorClient(commander.binary, commander.endpoint, commander.region, - 'darwin', keyParam, commander.publisherProofKey) - await packageTorClient(commander.binary, commander.endpoint, commander.region, - 'linux', keyParam, commander.publisherProofKey) - await packageTorClient(commander.binary, commander.endpoint, commander.region, - 'linux-arm64', keyParam, commander.publisherProofKey) - await packageTorClient(commander.binary, commander.endpoint, commander.region, - 'win32', keyParam, commander.publisherProofKey) -}) +const args = getPackagingArgs() +Promise.all(platforms.map(platform => packageComponent(args, new TorClient(platform)))) diff --git a/scripts/packageTorPluggableTransports.js b/scripts/packageTorPluggableTransports.js index 9750630d..66d1b70d 100644 --- a/scripts/packageTorPluggableTransports.js +++ b/scripts/packageTorPluggableTransports.js @@ -5,12 +5,12 @@ // Example usage: // npm run package-tor-pluggable-transports -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --keys-directory path/to/key/dir -import commander from 'commander' import { execSync } from 'child_process' import fs from 'fs' import { mkdirp } from 'mkdirp' import path from 'path' import util from '../lib/util.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' const TOR_PLUGGABLE_TRANSPORTS_UPDATER = 'tor-pluggable-transports-updater' @@ -74,44 +74,7 @@ class TorPluggableTransports { } } -const packageTorPluggableTransports = async (binary, endpoint, region, platform, key, publisherProofKey) => { - const descriptor = new TorPluggableTransports(platform) +const platforms = ['darwin', 'linux', 'win32'] - const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : descriptor.privateKeyFromDir(key) - - await util.prepareNextVersionCRX( - binary, - publisherProofKey, - endpoint, - region, - descriptor, - privateKeyFile, - false) -} - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files', 'abc') - .option('-f, --key-file ', 'private key file for signing crx', 'key.pem')) - .parse(process.argv) - -let keyParam = '' - -if (fs.existsSync(commander.keyFile)) { - keyParam = commander.keyFile -} else if (fs.existsSync(commander.keysDirectory)) { - keyParam = commander.keysDirectory -} else { - throw new Error('Missing or invalid private key file/directory') -} - -util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - await packageTorPluggableTransports(commander.binary, commander.endpoint, commander.region, - 'darwin', keyParam, commander.publisherProofKey) - await packageTorPluggableTransports(commander.binary, commander.endpoint, commander.region, - 'linux', keyParam, commander.publisherProofKey) - await packageTorPluggableTransports(commander.binary, commander.endpoint, commander.region, - 'win32', keyParam, commander.publisherProofKey) -}) +const args = getPackagingArgs() +Promise.all(platforms.map(platform => packageComponent(args, new TorPluggableTransports(platform)))) diff --git a/scripts/packageWalletDataFiles.js b/scripts/packageWalletDataFiles.js index d834f551..7755fc3f 100644 --- a/scripts/packageWalletDataFiles.js +++ b/scripts/packageWalletDataFiles.js @@ -5,10 +5,10 @@ // Example usage: // npm run package-wallet-data-files -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --key-file path/to/wallet-data-files-updater.pem -import commander from 'commander' import fs from 'fs-extra' import path from 'path' import util from '../lib/util.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' const getOriginalManifest = (packageDir) => { return path.join(packageDir, 'manifest.json') @@ -36,57 +36,5 @@ class WalletDataFilesUpdater { } } -const generateCRXFile = async (binary, endpoint, region, key, - publisherProofKey, localRun) => { - const descriptor = new WalletDataFilesUpdater() - - let privateKeyFile = '' - if (!localRun) { - privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : descriptor.privateKeyFromDir(key) - } - - await util.prepareNextVersionCRX( - binary, - publisherProofKey, - endpoint, - region, - descriptor, - privateKeyFile, - localRun) -} - -const processJob = async (commander, keyParam) => { - await generateCRXFile(commander.binary, commander.endpoint, - commander.region, keyParam, - commander.publisherProofKey, - commander.localRun) -} - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files') - .option('-f, --key-file ', 'private key file for signing crx', 'key.pem') - .option('-l, --local-run', 'Runs updater job without connecting anywhere remotely')) - .parse(process.argv) - -let keyParam = '' - -if (!commander.localRun) { - if (fs.existsSync(commander.keyFile)) { - keyParam = commander.keyFile - } else if (fs.existsSync(commander.keysDirectory)) { - keyParam = commander.keysDirectory - } else { - throw new Error('Missing or invalid private key file/directory') - } -} - -if (!commander.localRun) { - util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - await processJob(commander, keyParam) - }) -} else { - processJob(commander, keyParam) -} +const args = getPackagingArgs() +packageComponent(args, new WalletDataFilesUpdater())