diff --git a/lib/adBlockRustUtils.js b/lib/adBlockRustUtils.js index 74bb1363..ce1d5455 100644 --- a/lib/adBlockRustUtils.js +++ b/lib/adBlockRustUtils.js @@ -1,7 +1,8 @@ -import { Engine, FilterSet, uBlockResources } from 'adblock-rs' +import { Engine, FilterSet, RuleTypes, uBlockResources } from 'adblock-rs' +import fs from 'fs-extra' import path from 'path' -import { promises as fs } from 'fs' +import util from '../lib/util.js' const uBlockLocalRoot = 'submodules/uBlock' const uBlockWebAccessibleResources = path.join(uBlockLocalRoot, 'src/web_accessible_resources') @@ -50,9 +51,6 @@ const getListCatalog = lazyInit(async () => { // Legacy logic requires a distinction between default and regional lists. // This can be removed once DAT support is no longer needed by iOS. const isDefaultList = entry => entry.default_enabled && entry.hidden -const getDefaultLists = () => getListCatalog().then(catalog => { - return catalog.filter(isDefaultList) -}) const getRegionalLists = () => getListCatalog().then(catalog => { return catalog.filter(entry => !isDefaultList(entry)) }) @@ -126,6 +124,72 @@ const generateResourcesFile = async (outLocation) => { return fs.writeFile(outLocation, await generateResources(), 'utf8') } +// Removes Brave-specific scriptlet injections from non-Brave lists +const enforceBraveDirectives = (title, data) => { + if (!title || !title.startsWith('Brave ')) { + return data.split('\n').filter(line => { + const hasBraveScriptlet = line.indexOf('+js(brave-') >= 0 + if (hasBraveScriptlet) { + console.log('List ' + title + ' attempted to include brave-specific directive: ' + line) + } + return !hasBraveScriptlet + }).join('\n') + } else { + return data + } +} + +/** + * Parses the passed in filter rule data and serializes a data file to disk. + * + * @param filterRuleData An array of { format, data, includeRedirectUrls, ruleTypes } where format is one of `adblock-rust`'s supported filter parsing formats and data is a newline-separated list of such filters. + * includeRedirectUrls is a boolean: https://github.com/brave/adblock-rust/pull/184. We only support redirect URLs on filter lists we maintain and trust. + * ruleTypes was added with https://github.com/brave/brave-core-crx-packager/pull/298 and allows for { RuleTypes.ALL, RuleTypes.NETWORK_ONLY, RuleTypes.COSMETIC_ONLY } + * @param outputDATFilename The filename of the DAT file to create. + */ +const generateDataFileFromLists = (filterRuleData, outPath, defaultRuleType = RuleTypes.ALL) => { + const filterSet = new FilterSet(false) + for (let { title, format, data, includeRedirectUrls, ruleTypes } of filterRuleData) { + includeRedirectUrls = Boolean(includeRedirectUrls) + ruleTypes = ruleTypes || defaultRuleType + const parseOpts = { format, includeRedirectUrls, ruleTypes } + filterSet.addFilters(enforceBraveDirectives(title, data).split('\n'), parseOpts) + } + const client = new Engine(filterSet, true) + const arrayBuffer = client.serializeRaw() + fs.writeFileSync(outPath, Buffer.from(arrayBuffer)) +} + +/** + * Serializes the provided lists to disk in one file as `list.txt` under the given component subdirectory. + */ +const generatePlaintextListFromLists = (listBuffers, outPath) => { + const fullList = listBuffers.map(({ data, title }) => enforceBraveDirectives(title, data)).join('\n') + fs.writeFileSync(outPath, fullList) +} + +/** + * Returns a promise that resolves to the contents of each list from the entry's sources. + * Throws if any of the lists fail the sanity check. + */ +const downloadListsForEntry = (entry) => { + const lists = entry.sources + + const promises = [] + lists.forEach((l) => { + console.log(`${entry.langs} ${l.url}...`) + promises.push(util.fetchTextFromURL(l.url) + .then(data => ({ title: l.title || entry.title, format: l.format, data })) + .then(listBuffer => { + sanityCheckList(listBuffer) + return listBuffer + }) + ) + }) + + return Promise.all(promises) +} + /** * A list of requests that should not be blocked unless the list has some serious issue. * @@ -178,13 +242,15 @@ const sanityCheckList = ({ title, format, data }) => { } export { + downloadListsForEntry, regionalCatalogComponentId, regionalCatalogPubkey, resourcesComponentId, resourcesPubkey, sanityCheckList, + generateDataFileFromLists, + generatePlaintextListFromLists, generateResourcesFile, getListCatalog, - getDefaultLists, getRegionalLists } diff --git a/lib/util.js b/lib/util.js index 31d5b2b6..c3ca3ed1 100644 --- a/lib/util.js +++ b/lib/util.js @@ -64,6 +64,49 @@ const fetchTextFromURL = (listURL) => { return p } +const prepareNextVersionCRX = async (binary, publisherProofKey, endpoint, region, componentDescriptor, privateKeyFile, localRun) => { + mkdirp.sync(componentDescriptor.stagingDir) + + let contentHash + if (componentDescriptor.contentHash !== undefined) { + try { + contentHash = componentDescriptor.contentHash() + } catch (_) { + // ignore failure if e.g. previous version does not exist yet + } + } + + let version + if (!localRun) { + version = '1.0.0' + } else { + version = await getNextVersion(endpoint, region, componentDescriptor.componentId, contentHash) + if (version === undefined) { + console.log(`content for ${componentDescriptor.componentId} was not updated, skipping!`) + return + } + } + + const contentHashFile = componentDescriptor.contentHashFile + + // Remove any existing `.contentHash` file for determinism + if (contentHashFile !== undefined && fs.existsSync(contentHashFile)) { + fs.unlinkSync(contentHashFile) + } + + await componentDescriptor.stageFiles(version, componentDescriptor.stagingDir) + + if (!localRun) { + generateCRXFile(binary, componentDescriptor.crxFile, privateKeyFile, publisherProofKey, componentDescriptor.stagingDir) + } + + if (contentHash !== undefined && contentHashFile !== undefined) { + fs.writeFileSync(contentHashFile, contentHash) + } + + console.log(`Generated ${componentDescriptor.crxFile} with version number ${version}`) +} + const generateCRXFile = (binary, crxFile, privateKeyFile, publisherProofKey, inputDir) => { if (!binary) { @@ -76,8 +119,12 @@ const generateCRXFile = (binary, crxFile, privateKeyFile, publisherProofKey, throw new Error(`Private key file '${privateKeyFile}' is missing, was it uploaded?`) } + const crxOutputDir = path.dirname(crxFile) + mkdirp.sync(crxOutputDir) + const tmp = tmpdir() const tempUserDataDir = fs.mkdtempSync(path.join(tmp, 'crx-package-job-')) + const args = [ `--pack-extension=${path.resolve(inputDir)}`, `--pack-extension-key=${path.resolve(privateKeyFile)}`, @@ -551,18 +598,6 @@ const updateDynamoDB = (endpoint, region, id, version, hash, name, disabled, con return dynamodb.send(new PutItemCommand(params)) } -const addCommonScriptOptions = (command) => { - return command - .option('-b, --binary ', - 'Path to the Chromium based executable to use to generate the CRX file') - .option('-p, --publisher-proof-key ', - 'File containing private key for generating publisher proof') - - // If setup locally, use --endpoint http://localhost:8000 - .option('-e, --endpoint ', 'DynamoDB endpoint to connect to', '') - .option('-r, --region ', 'The AWS region to use', 'us-west-2') -} - const escapeStringForJSON = str => { if (typeof str !== 'string') { throw new Error('Not a string: ' + JSON.stringify(str)) @@ -613,18 +648,16 @@ export default { fetchTextFromURL, createTableIfNotExists, fetchPreviousVersions, - generateCRXFile, generatePuffPatches, generateSHA256HashOfFile, - getNextVersion, getIDFromBase64PublicKey, installErrorHandlers, parseManifest, uploadCRXFile, updateDBForCRXFile, - addCommonScriptOptions, escapeStringForJSON, copyManifestWithVersion, stageDir, - stageFiles + stageFiles, + prepareNextVersionCRX } diff --git a/package.json b/package.json index 72fc3e5f..157dceb0 100644 --- a/package.json +++ b/package.json @@ -39,12 +39,12 @@ "generate-user-model-installer-updates": "node scripts/generateBraveAdsResourcesComponentInputFiles.js", "lint": "standard lib scripts test", "package-brave-player": "node ./scripts/packageBravePlayer", - "package-ethereum-remote-client": "node ./scripts/packageComponent --type ethereum-remote-client", + "package-ethereum-remote-client": "node ./scripts/packageEthereumRemoteClient", "package-ad-block": "node ./scripts/packageAdBlock", "package-tor-client": "node ./scripts/packageTorClient", "package-tor-pluggable-transports": "node ./scripts/packageTorPluggableTransports", "package-ipfs-daemon": "node ./scripts/packageIpfsDaemon", - "package-wallet-data-files": "node ./scripts/packageComponent --type wallet-data-files-updater", + "package-wallet-data-files": "node ./scripts/packageWalletDataFiles", "package-local-data-files": "node ./scripts/packageLocalDataFiles", "package-ntp-background-images": "node ./scripts/packageNTPBackgroundImagesComponent", "package-ntp-sponsored-images": "node ./scripts/ntp-sponsored-images/package", diff --git a/scripts/generateAdBlockRustDataFiles.js b/scripts/generateAdBlockRustDataFiles.js index b46977c9..b1dc0a67 100644 --- a/scripts/generateAdBlockRustDataFiles.js +++ b/scripts/generateAdBlockRustDataFiles.js @@ -2,143 +2,5 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { - generateResourcesFile, - getListCatalog, - getDefaultLists, - getRegionalLists, - resourcesComponentId, - regionalCatalogComponentId, - sanityCheckList -} from '../lib/adBlockRustUtils.js' -import Sentry from '../lib/sentry.js' -import util from '../lib/util.js' -import path from 'path' -import fs from 'fs' - -/** - * Obtains the output path to store a file given the specied name and subdir - */ -const getOutPath = (outputFilename, outSubdir) => { - let outPath = path.join('build') - if (!fs.existsSync(outPath)) { - fs.mkdirSync(outPath) - } - outPath = path.join(outPath, 'ad-block-updater') - if (!fs.existsSync(outPath)) { - fs.mkdirSync(outPath) - } - outPath = path.join(outPath, outSubdir) - if (!fs.existsSync(outPath)) { - fs.mkdirSync(outPath) - } - return path.join(outPath, outputFilename) -} - -// Removes Brave-specific scriptlet injections from non-Brave lists -const enforceBraveDirectives = (title, data) => { - if (!title || !title.startsWith('Brave ')) { - return data.split('\n').filter(line => { - const hasBraveScriptlet = line.indexOf('+js(brave-') >= 0 - if (hasBraveScriptlet) { - console.log('List ' + title + ' attempted to include brave-specific directive: ' + line) - } - return !hasBraveScriptlet - }).join('\n') - } else { - return data - } -} - -/** - * Serializes the provided lists to disk in one file as `list.txt` under the given component subdirectory. - */ -const generatePlaintextListFromLists = (listBuffers, outSubdir) => { - const fullList = listBuffers.map(({ data, title }) => enforceBraveDirectives(title, data)).join('\n') - fs.writeFileSync(getOutPath('list.txt', outSubdir), fullList) -} - -/** - * Convenience function that generates component files for a given catalog entry. - * - * If any list source cannot be downloaded, the promise will resolve but the new files will _not_ be generated. - * - * @param entry the corresponding entry directly from one of Brave's list catalogs - * @param doIos boolean, whether or not filters for iOS should be created (currently only used by default list) - * @return a Promise which resolves upon completion - */ -const generateDataFilesForCatalogEntry = (entry) => { - const lists = entry.sources - - const promises = [] - lists.forEach((l) => { - console.log(`${entry.langs} ${l.url}...`) - promises.push(util.fetchTextFromURL(l.url) - .then(data => ({ title: l.title || entry.title, format: l.format, data })) - .then(listBuffer => { - sanityCheckList(listBuffer) - return listBuffer - }) - ) - }) - return Promise.all(promises) - .then( - listBuffers => generatePlaintextListFromLists(listBuffers, entry.list_text_component.component_id), - e => { - console.error(`Not publishing a new version of ${entry.title} due to failure downloading a source: ${e.message}`) - if (Sentry) { - Sentry.captureException(e, { level: 'warning' }) - } - } - ) -} - -/** - * Convenience function that generates a DAT file for each region, and writes - * the catalog of available regional lists to the default list directory and - * regional catalog component directory. - */ -const generateDataFilesForAllRegions = () => { - console.log('Processing per region list updates...') - return getRegionalLists().then(regions => { - return new Promise((resolve, reject) => { - const catalogString = JSON.stringify(regions) - fs.writeFileSync(getOutPath('regional_catalog.json', regionalCatalogComponentId), catalogString) - getListCatalog().then(listCatalog => { - const catalogString = JSON.stringify(listCatalog) - fs.writeFileSync(getOutPath('list_catalog.json', regionalCatalogComponentId), catalogString) - resolve() - }) - }).then(() => Promise.all(regions.map(region => - generateDataFilesForCatalogEntry(region) - ))) - }) -} - -const generateDataFilesForResourcesComponent = () => { - return generateResourcesFile(getOutPath('resources.json', resourcesComponentId)) -} - -const generateDataFilesForDefaultAdblock = () => getDefaultLists() - .then(defaultLists => Promise.all(defaultLists.map(list => generateDataFilesForCatalogEntry(list)))) - -generateDataFilesForDefaultAdblock() - .then(generateDataFilesForResourcesComponent) - .then(generateDataFilesForAllRegions) - .then(() => { - console.log('Thank you for updating the data files, don\'t forget to upload them too!') - }) - .catch((e) => { - console.error(`Something went wrong, aborting: ${e} ${e.stack} ${e.message}`) - process.exit(1) - }) - -process.on('uncaughtException', (err) => { - console.error('Caught exception:', err) - process.exit(1) -}) - -process.on('unhandledRejection', (err) => { - console.error('Unhandled rejection:', err) - process.exit(1) -}) +// TODO remove this after the Jenkins job is updated +console.log('Data files for adblock-rust are now generated as part of the package-ad-block job.') diff --git a/scripts/generateManifestForRustAdblock.js b/scripts/generateManifestForRustAdblock.js index 2a8ecaa6..0af1a7af 100644 --- a/scripts/generateManifestForRustAdblock.js +++ b/scripts/generateManifestForRustAdblock.js @@ -2,59 +2,5 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { promises as fs } from 'fs' -import path from 'path' - -import { getListCatalog, regionalCatalogComponentId, regionalCatalogPubkey, resourcesComponentId, resourcesPubkey } from '../lib/adBlockRustUtils.js' -import util from '../lib/util.js' - -const outPath = path.join('build', 'ad-block-updater') - -const generateManifestFile = async (name, base64PublicKey, subdir) => { - const manifest = '{\n' + - ' "description": "Brave Ad Block Updater extension",\n' + - ' "key": "' + base64PublicKey + '",\n' + - ' "manifest_version": 2,\n' + - ' "name": "Brave Ad Block Updater (' + name + ')",\n' + - ' "version": "0.0.0"\n' + - '}\n' - - const filePath = path.join(outPath, subdir, 'manifest.json') - return fs.writeFile(filePath, manifest) - .catch(e => console.warn('Skipped writing manifest for ' + name + ': ' + e.message)) -} - -const generateManifestFileForRegionalCatalog = - generateManifestFile.bind(null, 'Regional Catalog', regionalCatalogPubkey, regionalCatalogComponentId) - -const generateManifestFileForResources = - generateManifestFile.bind(null, 'Resources', resourcesPubkey, resourcesComponentId) - -const generateManifestFilesForAllLists = async () => { - const catalog = await getListCatalog() - return Promise.all(catalog.map(async entry => { - const title = util.escapeStringForJSON(entry.title) - await generateManifestFile(title + ' (plaintext)', entry.list_text_component.base64_public_key, entry.list_text_component.component_id) - })) -} - -generateManifestFileForRegionalCatalog() - .then(generateManifestFileForResources) - .then(generateManifestFilesForAllLists) - .then(() => { - console.log('Thank you for updating the data files, don\'t forget to upload them too!') - }) - .catch((e) => { - console.error(`Something went wrong, aborting: ${e}`) - process.exit(1) - }) - -process.on('uncaughtException', (err) => { - console.error('Caught exception:', err) - process.exit(1) -}) - -process.on('unhandledRejection', (err) => { - console.error('Unhandled rejection:', err) - process.exit(1) -}) +// TODO remove this after the Jenkins job is updated +console.log('Manifests for adblock-rust are now generated as part of the package-ad-block job.') diff --git a/scripts/ntp-sponsored-images/package.js b/scripts/ntp-sponsored-images/package.js index bcd18227..eaa7a2e0 100644 --- a/scripts/ntp-sponsored-images/package.js +++ b/scripts/ntp-sponsored-images/package.js @@ -3,26 +3,20 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // you can obtain one at http://mozilla.org/MPL/2.0/. -import commander from 'commander' import fs from 'fs-extra' import { mkdirp } from 'mkdirp' import path from 'path' import util from '../../lib/util.js' import params from './params.js' +import { getPackagingArgs, packageComponent } from './packageComponent' -const stageFiles = (locale, version, outputDir) => { - util.stageDir( - path.join(path.resolve(), 'build', 'ntp-sponsored-images', 'resources', locale, '/'), - getManifestPath(locale), - version, - outputDir) -} +const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-sponsored-images') -const generateManifestFile = (regionPlatform, componentData) => { +const generateManifestFile = (regionPlatform, publicKey) => { const manifestPath = getManifestPath(regionPlatform) const manifestContent = { description: `Brave NTP sponsored images component (${regionPlatform})`, - key: componentData.key, + key: publicKey, manifest_version: 2, name: 'Brave NTP sponsored images', version: '0.0.0' @@ -46,48 +40,40 @@ function getManifestPath (regionPlatform) { return path.join(getManifestsDir(), `${regionPlatform}-manifest.json`) } -const generateCRXFile = (binary, endpoint, region, keyDir, platformRegion, - componentData, publisherProofKey) => { - const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-sponsored-images') - const stagingDir = path.join(rootBuildDir, 'staging', platformRegion) - const crxOutputDir = path.join(rootBuildDir, 'output') - mkdirp.sync(stagingDir) - mkdirp.sync(crxOutputDir) - util.getNextVersion(endpoint, region, componentData.id).then((version) => { - const crxFile = path.join(crxOutputDir, `ntp-sponsored-images-${platformRegion}.crx`) - // Desktop private key file names do not have the -desktop suffix, but android has -android - const privateKeyFile = path.join(keyDir, `ntp-sponsored-images-${platformRegion.replace('-desktop', '')}.pem`) - stageFiles(platformRegion, version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) -} +class NtpSponsoredImages { + constructor (platformRegion, componentData) { + this.platformRegion = platformRegion + this.locale = componentData.locale + this.publicKey = componentData.key + this.componentId = componentData.id -util.installErrorHandlers() + this.stagingDir = path.join(rootBuildDir, 'staging', this.platformRegion) + this.crxFile = path.join(rootBuildDir, 'output', `ntp-sponsored-images-${this.platformRegion}.crx`) + } -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files') - .option('-t, --target-regions ', 'Comma separated list of regions that should generate SI component. For example: "AU-android,US-desktop,GB-ios"', '') - .option('-u, --excluded-target-regions ', 'Comma separated list of regions that should not generate SI component. For example: "AU-android,US-desktop,GB-ios"', '')) - .parse(process.argv) + privateKeyFromDir (keyDir) { + // Desktop private key file names do not have the -desktop suffix, but android has -android + return path.join(keyDir, `ntp-sponsored-images-${this.platformRegion.replace('-desktop', '')}.pem`) + } -let keyDir = '' -if (fs.existsSync(commander.keysDirectory)) { - keyDir = commander.keysDirectory -} else { - throw new Error('Missing or invalid private key directory') + async stageFiles (version, outputDir) { + generateManifestFile(this.platformRegion, this.publicKey) + util.stageDir( + path.join(path.resolve(), 'build', 'ntp-sponsored-images', 'resources', this.locale, '/'), + getManifestPath(this.locale), + version, + outputDir) + } } -const targetComponents = params.getTargetComponents(commander.targetRegions, commander.excludedTargetRegions) +const args = getPackagingArgs([ + ['-t, --target-regions ', 'Comma separated list of regions that should generate SI component. For example: "AU-android,US-desktop,GB-ios"', ''], + ['-u, --excluded-target-regions ', 'Comma separated list of regions that should not generate SI component. For example: "AU-android,US-desktop,GB-ios"', ''] +]) -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - for (const platformRegion of Object.keys(targetComponents)) { - const componentData = targetComponents[platformRegion] - generateManifestFile(platformRegion, componentData) - generateCRXFile(commander.binary, commander.endpoint, commander.region, - keyDir, platformRegion, componentData, - commander.publisherProofKey) - } -}) +const targetComponents = params.getTargetComponents(args.targetRegions, args.excludedTargetRegions) + +for (const platformRegion of Object.keys(targetComponents)) { + const componentData = targetComponents[platformRegion] + packageComponent(args, new NtpSponsoredImages(platformRegion, componentData)) +} diff --git a/scripts/packageAdBlock.js b/scripts/packageAdBlock.js index d2d3010d..7f1dfdbd 100644 --- a/scripts/packageAdBlock.js +++ b/scripts/packageAdBlock.js @@ -6,122 +6,179 @@ // Example usage: // npm run package-ad-block -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --key-file path/to/ad-block-updater-regional-component-keys -import commander from 'commander' import fs from 'fs-extra' import path from 'path' +import Sentry from '../lib/sentry.js' import util from '../lib/util.js' -import { getListCatalog, regionalCatalogComponentId, resourcesComponentId } from '../lib/adBlockRustUtils.js' - -async function stageFiles (version, outputDir) { - // ad-block components are already written in the output directory - // so we don't need to stage anything - const originalManifest = path.join(outputDir, 'manifest.json') - // note - in-place manifest replacement, unlike other components - util.copyManifestWithVersion(originalManifest, outputDir, version) -} - -const postNextVersionWork = (componentSubdir, key, publisherProofKey, - binary, localRun, version, contentHash) => { - const stagingDir = path.join('build', 'ad-block-updater', componentSubdir) - const crxOutputDir = path.join('build', 'ad-block-updater') - const crxFile = path.join(crxOutputDir, `ad-block-updater-${componentSubdir}.crx`) - const contentHashFile = path.join(crxOutputDir, `ad-block-updater-${componentSubdir}.contentHash`) - stageFiles(version, stagingDir).then(() => { - // Remove any existing `.contentHash` file for determinism - if (fs.existsSync(contentHashFile)) { - fs.unlinkSync(contentHashFile) - } - if (!localRun) { - const privateKeyFile = path.join(key, `ad-block-updater-${componentSubdir}.pem`) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - } - if (contentHash !== undefined) { - fs.writeFileSync(contentHashFile, contentHash) - } - console.log(`Generated ${crxFile} with version number ${version}`) - }) -} +import { getPackagingArgs, packageComponent } from './packageComponent.js' +import { + downloadListsForEntry, + generatePlaintextListFromLists, + generateResourcesFile, + getListCatalog, + getRegionalLists, + resourcesComponentId, + resourcesPubkey, + regionalCatalogComponentId, + regionalCatalogPubkey +} from '../lib/adBlockRustUtils.js' const getOriginalManifest = (componentSubdir) => { const manifestsDir = path.join('build', 'ad-block-updater') return path.join(manifestsDir, componentSubdir, 'manifest.json') } -const processComponent = (binary, endpoint, region, keyDir, - publisherProofKey, localRun, componentSubdir) => { - const originalManifest = getOriginalManifest(componentSubdir) - const parsedManifest = util.parseManifest(originalManifest) - const id = util.getIDFromBase64PublicKey(parsedManifest.key) - - let fileToHash - if (componentSubdir === regionalCatalogComponentId) { - fileToHash = 'regional_catalog.json' - } else if (componentSubdir === resourcesComponentId) { - fileToHash = 'resources.json' - } else { - fileToHash = 'list.txt' +const generateManifestFile = async (name, base64PublicKey, componentSubdir) => { + const manifest = '{\n' + + ' "description": "Brave Ad Block Updater extension",\n' + + ' "key": "' + base64PublicKey + '",\n' + + ' "manifest_version": 2,\n' + + ' "name": "Brave Ad Block Updater (' + name + ')",\n' + + ' "version": "0.0.0"\n' + + '}\n' + + const filePath = getOriginalManifest(componentSubdir) + return fs.writeFile(filePath, manifest) + .catch(e => console.warn('Skipped writing manifest for ' + name + ': ' + e.message)) +} + +// Serves plaintext filter lists +class AdblockList { + constructor (catalogEntry) { + this.catalogEntry = catalogEntry + this.componentId = catalogEntry.list_text_component.component_id + + this.crxName = `ad-block-updater-${this.componentId}` + this.stagingDir = path.join('build', 'ad-block-updater', this.componentId) + this.crxFile = path.join('build', 'ad-block-updater', `${this.crxName}.crx`) + this.contentHashFile = path.join('build', 'ad-block-updater', `${this.crxName}.contentHash`) } - let contentHash - if (fileToHash !== undefined) { - const contentFile = path.join('build', 'ad-block-updater', componentSubdir, fileToHash) - contentHash = util.generateSHA256HashOfFile(contentFile) + contentHash () { + const contentFile = path.join('build', 'ad-block-updater', this.componentId, 'list.txt') + return util.generateSHA256HashOfFile(contentFile) } - if (!localRun) { - util.getNextVersion(endpoint, region, id, contentHash).then((version) => { - if (version !== undefined) { - postNextVersionWork(componentSubdir, keyDir, publisherProofKey, - binary, localRun, version, contentHash) - } else { - console.log('content for ' + id + ' was not updated, skipping!') + privateKeyFromDir (keyDir) { + return path.join(keyDir, `${this.crxName}.pem`) + } + + async stageFiles (version, outputDir) { + await downloadListsForEntry(this.catalogEntry).then( + (listBuffers) => { + generatePlaintextListFromLists(listBuffers, path.join(outputDir, 'list.txt')) + }, + e => { + console.error(`Not publishing a new version of ${this.catalogEntry.title} due to failure downloading a source: ${e.message}`) + if (Sentry) { + Sentry.captureException(e, { level: 'warning' }) + } } - }) - } else { - postNextVersionWork(componentSubdir, undefined, publisherProofKey, - binary, localRun, '1.0.0', contentHash) + ) + + const title = util.escapeStringForJSON(this.catalogEntry.title) + if (this.catalogEntry.list_text_component) { + await generateManifestFile(title + ' (plaintext)', this.catalogEntry.list_text_component.base64_public_key, this.componentId) + } + + const files = [ + { path: getOriginalManifest(this.componentId) }, + { path: path.join(outputDir, 'list.txt') } + ] + util.stageFiles(files, version, outputDir) } } -const getComponentList = async () => { - const output = [ - regionalCatalogComponentId, - resourcesComponentId - ] - const catalog = await getListCatalog() - catalog.forEach(entry => { - output.push(entry.list_text_component.component_id) - }) - return output -} +// Serves the catalog of available filter lists +// Currently there are two files: +// - `regional_catalog.json` is the older version +// - `list_catalog.json` is the newer version, containing additional list metadata +class AdblockCatalog { + constructor () { + this.componentId = regionalCatalogComponentId + + this.crxName = `ad-block-updater-${this.componentId}` + this.stagingDir = path.join('build', 'ad-block-updater', this.componentId) + this.crxFile = path.join('build', 'ad-block-updater', `${this.crxName}.crx`) + this.contentHashFile = path.join('build', 'ad-block-updater', `${this.crxName}.contentHash`) + } -const processJob = async (commander, keyDir) => { - (await getComponentList()) - .forEach(processComponent.bind(null, commander.binary, commander.endpoint, - commander.region, keyDir, - commander.publisherProofKey, - commander.localRun)) + contentHash () { + // TODO update to `list_catalog.json` when `regional_catalog.json` is no longer used + const contentFile = path.join('build', 'ad-block-updater', this.componentId, 'regional_catalog.json') + return util.generateSHA256HashOfFile(contentFile) + } + + privateKeyFromDir (keyDir) { + return path.join(keyDir, `${this.crxName}.pem`) + } + + async stageFiles (version, outputDir) { + const regions = await getRegionalLists() + const regionalCatalogString = JSON.stringify(regions) + const regionalCatalogPath = path.join(outputDir, 'regional_catalog.json') + fs.writeFileSync(regionalCatalogPath, regionalCatalogString) + + const listCatalog = await getListCatalog() + const listCatalogString = JSON.stringify(listCatalog) + const listCatalogPath = path.join(outputDir, 'list_catalog.json') + fs.writeFileSync(listCatalogPath, listCatalogString) + + await generateManifestFile('Regional Catalog', regionalCatalogPubkey, regionalCatalogComponentId) + + const files = [ + { path: getOriginalManifest(this.componentId) }, + { path: regionalCatalogPath }, + { path: listCatalogPath } + ] + util.stageFiles(files, version, outputDir) + } } -util.installErrorHandlers() +// Serves the data for adblock-rust's resource replacements and scriptlets. +class AdblockResources { + constructor () { + this.componentId = resourcesComponentId -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files') - .option('-l, --local-run', 'Runs updater job without connecting anywhere remotely')) - .parse(process.argv) + this.crxName = `ad-block-updater-${this.componentId}` + this.stagingDir = path.join('build', 'ad-block-updater', this.componentId) + this.crxFile = path.join('build', 'ad-block-updater', `${this.crxName}.crx`) + this.contentHashFile = path.join('build', 'ad-block-updater', `${this.crxName}.contentHash`) + } + + contentHash () { + const contentFile = path.join('build', 'ad-block-updater', this.componentId, 'resources.json') + return util.generateSHA256HashOfFile(contentFile) + } -if (!commander.localRun) { - let keyDir = '' - if (fs.existsSync(commander.keysDirectory)) { - keyDir = commander.keysDirectory - } else { - throw new Error('Missing or invalid private key file/directory') + privateKeyFromDir (keyDir) { + return path.join(keyDir, `${this.crxName}.pem`) } - util.createTableIfNotExists(commander.endpoint, commander.region).then(async () => { - await processJob(commander, keyDir) - }) -} else { - processJob(commander, undefined) + + async stageFiles (version, outputDir) { + const resourcesPath = path.join(outputDir, 'resources.json') + await generateResourcesFile(resourcesPath) + + await generateManifestFile('Resources', resourcesPubkey, resourcesComponentId) + + const files = [ + { path: getOriginalManifest(this.componentId) }, + { path: resourcesPath } + ] + util.stageFiles(files, version, outputDir) + } +} + +const getAdblockComponentDescriptors = async () => { + const output = [ + new AdblockCatalog(), + new AdblockResources() + ] + output.push(...(await getListCatalog()).map(entry => new AdblockList(entry))) + return output } + +const args = getPackagingArgs() +await Promise.all((await getAdblockComponentDescriptors()).map(descriptor => + packageComponent(args, descriptor) +)) diff --git a/scripts/packageBraveAdsResourcesComponent.js b/scripts/packageBraveAdsResourcesComponent.js index 5f1fa32c..89fea4a0 100644 --- a/scripts/packageBraveAdsResourcesComponent.js +++ b/scripts/packageBraveAdsResourcesComponent.js @@ -2,11 +2,11 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ -import commander from 'commander' import fs from 'fs-extra' import { mkdirp } from 'mkdirp' import path from 'path' import util from '../lib/util.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' const getComponentDataList = () => { return [ @@ -233,29 +233,7 @@ const getComponentDataList = () => { ] } -const stageFiles = (locale, version, outputDir) => { - util.stageDir( - path.join(path.resolve(), 'build', 'user-model-installer', 'resources', locale, '/'), - getOriginalManifest(locale), - version, - outputDir) -} - -const generateManifestFile = (componentData) => { - const manifestFile = getOriginalManifest(componentData.locale) - const manifestContent = { - description: 'Brave Ads Resources Component', - key: componentData.key, - manifest_version: 2, - name: 'Brave Ads Resources', - version: '0.0.0' - } - fs.writeFileSync(manifestFile, JSON.stringify(manifestContent)) -} - -const generateManifestFiles = () => { - getComponentDataList().forEach(generateManifestFile) -} +const rootBuildDir = path.join(path.resolve(), 'build', 'user-model-installer') const getManifestsDir = () => { const targetResourceDir = path.join(path.resolve(), 'build', 'user-model-installer', 'manifiest-files') @@ -267,42 +245,43 @@ const getOriginalManifest = (locale) => { return path.join(getManifestsDir(), `${locale}-manifest.json`) } -const generateCRXFile = (binary, endpoint, region, keyDir, publisherProofKey, - componentData) => { - const locale = componentData.locale - const rootBuildDir = path.join(path.resolve(), 'build', 'user-model-installer') - const stagingDir = path.join(rootBuildDir, 'staging', locale) - const crxOutputDir = path.join(rootBuildDir, 'output') - mkdirp.sync(stagingDir) - mkdirp.sync(crxOutputDir) - util.getNextVersion(endpoint, region, componentData.id).then((version) => { - const crxFile = path.join(crxOutputDir, `user-model-installer-${locale}.crx`) - const privateKeyFile = path.join(keyDir, `user-model-installer-${locale}.pem`) - stageFiles(locale, version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) +const generateManifestFile = (locale, key) => { + const manifestFile = getOriginalManifest(locale) + const manifestContent = { + description: 'Brave Ads Resources Component', + key, + manifest_version: 2, + name: 'Brave Ads Resources', + version: '0.0.0' + } + fs.writeFileSync(manifestFile, JSON.stringify(manifestContent)) } -util.installErrorHandlers() +class BraveAdsResourcesComponent { + constructor (componentData) { + this.locale = componentData.locale + this.publicKey = componentData.key + this.componentId = componentData.id + this.stagingDir = path.join(rootBuildDir, 'staging', this.locale) + this.crxFile = path.join(rootBuildDir, 'output', `user-model-installer-${this.locale}.crx`) + } -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files')) - .parse(process.argv) + privateKeyFromDir (keyDir) { + return path.join(keyDir, `user-model-installer-${this.locale}.pem`) + } -let keyDir = '' -if (fs.existsSync(commander.keysDirectory)) { - keyDir = commander.keysDirectory -} else { - throw new Error('Missing or invalid private key directory') + async stageFiles (version, outputDir) { + generateManifestFile(this.locale, this.key) + util.stageDir( + path.join(path.resolve(), 'build', 'user-model-installer', 'resources', this.locale, '/'), + getOriginalManifest(this.locale), + version, + outputDir) + } } -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - generateManifestFiles() - getComponentDataList().forEach( - generateCRXFile.bind(null, commander.binary, commander.endpoint, - commander.region, keyDir, - commander.publisherProofKey)) -}) +const args = getPackagingArgs() + +await Promise.all(getComponentDataList().map(componentData => + packageComponent(args, new BraveAdsResourcesComponent(componentData)) +)) diff --git a/scripts/packageBravePlayer.js b/scripts/packageBravePlayer.js index cebc3b58..1a5d7441 100644 --- a/scripts/packageBravePlayer.js +++ b/scripts/packageBravePlayer.js @@ -10,81 +10,32 @@ // Example usage: // npm run package-brave-player -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --key-file path/to/brave-player.pem -import commander from 'commander' -import fs from 'fs-extra' import path from 'path' import util from '../lib/util.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' const getOriginalManifest = () => { return path.join('component-data', 'brave-player', 'manifest.json') } -const stageFiles = (version, outputDir) => { - util.stageDir(path.join('component-data', 'brave-player'), getOriginalManifest(), version, outputDir) -} +class BravePlayer { + constructor () { + const originalManifest = getOriginalManifest() + const parsedManifest = util.parseManifest(originalManifest) + this.componentId = util.getIDFromBase64PublicKey(parsedManifest.key) -const postNextVersionWork = (key, publisherProofKey, binary, localRun, version) => { - const stagingDir = path.join('build', 'brave-player') - const crxOutputDir = path.join('build') - const crxFile = path.join(crxOutputDir, 'brave-player.crx') - let privateKeyFile = '' - if (!localRun) { - privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, 'brave-player.pem') - } - stageFiles(version, stagingDir) - if (!localRun) { - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) + this.stagingDir = path.join('build', 'brave-player') + this.crxFile = path.join('build', 'brave-player.crx') } - console.log(`Generated ${crxFile} with version number ${version}`) -} -const processDATFile = (binary, endpoint, region, key, publisherProofKey, localRun) => { - const originalManifest = getOriginalManifest() - const parsedManifest = util.parseManifest(originalManifest) - const id = util.getIDFromBase64PublicKey(parsedManifest.key) - - if (!localRun) { - util.getNextVersion(endpoint, region, id).then((version) => { - postNextVersionWork(key, publisherProofKey, - binary, localRun, version) - }) - } else { - postNextVersionWork(key, publisherProofKey, - binary, localRun, '1.0.0') + privateKeyFromDir (keyDir) { + return path.join(keyDir, 'brave-player.pem') } -} -const processJob = (commander, keyParam) => { - processDATFile(commander.binary, commander.endpoint, commander.region, - keyParam, commander.publisherProofKey, commander.localRun) -} - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files') - .option('-f, --key-file ', 'private key file for signing crx', 'key.pem') - .option('-l, --local-run', 'Runs updater job without connecting anywhere remotely')) - .parse(process.argv) - -let keyParam = '' - -if (!commander.localRun) { - if (fs.existsSync(commander.keyFile)) { - keyParam = commander.keyFile - } else if (fs.existsSync(commander.keysDirectory)) { - keyParam = commander.keysDirectory - } else { - throw new Error('Missing or invalid private key file/directory') + async stageFiles (version, outputDir) { + util.stageDir(path.join('component-data', 'brave-player'), getOriginalManifest(), version, outputDir) } } -if (!commander.localRun) { - util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - processJob(commander, keyParam) - }) -} else { - processJob(commander, keyParam) -} +const args = getPackagingArgs() +packageComponent(args, new BravePlayer()) diff --git a/scripts/packageComponent.js b/scripts/packageComponent.js index ce66feea..2b9c4107 100644 --- a/scripts/packageComponent.js +++ b/scripts/packageComponent.js @@ -2,115 +2,57 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ -// Example usage: -// npm run package-ethereum-remote-client -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --key-file path/to/ethereum-remote-client.pem - import commander from 'commander' -import fs from 'fs-extra' -import path from 'path' +import fs from 'fs/promises' import util from '../lib/util.js' -const stageFiles = (componentType, version, outputDir) => { - util.stageDir(getPackageDirByComponentType(componentType), getOriginalManifest(componentType), version, outputDir) +const getPackagingArgs = (additionalParams) => { + util.installErrorHandlers() - if (componentType === 'wallet-data-files-updater') { - fs.unlinkSync(path.join(outputDir, 'package.json')) + let setup = commander + .option('-d, --keys-directory ', 'directory containing private keys for signing crx files') + .option('-f, --key-file ', 'private key file for signing crx') + .option('-l, --local-run', 'Runs updater job without connecting anywhere remotely') + .option('-b, --binary ', 'Path to the Chromium based executable to use to generate the CRX file') + .option('-p, --publisher-proof-key ', 'File containing private key for generating publisher proof') + .option('-e, --endpoint ', 'DynamoDB endpoint to connect to', '') + .option('-r, --region ', 'The AWS region to use', 'us-west-2') + + if (additionalParams !== undefined) { + for (const param of additionalParams) { + setup = setup.option(...param) + } } -} -const validComponentTypes = [ - 'ethereum-remote-client', - 'wallet-data-files-updater' -] - -const getPackageNameByComponentType = (componentType) => { - switch (componentType) { - case 'ethereum-remote-client': - return componentType - case 'wallet-data-files-updater': - return 'brave-wallet-lists' - default: - // shouldn't be possible to get here - return null - } -} -const getPackageDirByComponentType = (componentType) => { - return path.join('node_modules', getPackageNameByComponentType(componentType)) + return setup.parse(process.argv) } -const getOriginalManifest = (componentType) => { - return path.join(getPackageDirByComponentType(componentType), 'manifest.json') -} - -const postNextVersionWork = (componentType, key, publisherProofKey, - binary, localRun, version) => { - const stagingDir = path.join('build', componentType) - const crxFile = path.join(stagingDir, `${componentType}.crx`) +const packageComponent = async (packagingArgs, componentClass) => { let privateKeyFile = '' - if (!localRun) { - privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `${componentType}.pem`) - } - stageFiles(componentType, version, stagingDir) - if (!localRun) { - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - } - console.log(`Generated ${crxFile} with version number ${version}`) -} -const processDATFile = (binary, endpoint, region, componentType, key, - publisherProofKey, localRun) => { - const originalManifest = getOriginalManifest(componentType) - const parsedManifest = util.parseManifest(originalManifest) - const id = util.getIDFromBase64PublicKey(parsedManifest.key) - - if (!localRun) { - util.getNextVersion(endpoint, region, id).then((version) => { - postNextVersionWork(componentType, key, publisherProofKey, - binary, localRun, version) - }) - } else { - postNextVersionWork(componentType, key, publisherProofKey, - binary, localRun, '1.0.0') + if (packagingArgs.keyFile !== undefined && (await fs.lstat(packagingArgs.keyFile)).isFile()) { + privateKeyFile = packagingArgs.keyFile + } else if (packagingArgs.keysDirectory !== undefined && (await fs.lstat(packagingArgs.keysDirectory)).isDirectory()) { + privateKeyFile = componentClass.privateKeyFromDir(packagingArgs.keysDirectory) + } else if (packagingArgs.localRun !== true) { + throw new Error('Missing or invalid private key file/directory') } -} -const processJob = (commander, keyParam) => { - if (!validComponentTypes.includes(commander.type)) { - throw new Error('Unrecognized component extension type: ' + commander.type) + if (packagingArgs.localRun !== true) { + await util.createTableIfNotExists(packagingArgs.endpoint, packagingArgs.region) } - processDATFile(commander.binary, commander.endpoint, - commander.region, commander.type, keyParam, - commander.publisherProofKey, - commander.localRun) -} - -util.installErrorHandlers() -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files') - .option('-f, --key-file ', 'private key file for signing crx', 'key.pem') - .option('-t, --type ', 'component extension type', /^(local-data-files-updater|ethereum-remote-client|wallet-data-files-updater)$/i) - .option('-l, --local-run', 'Runs updater job without connecting anywhere remotely')) - .parse(process.argv) - -let keyParam = '' - -if (!commander.localRun) { - if (fs.existsSync(commander.keyFile)) { - keyParam = commander.keyFile - } else if (fs.existsSync(commander.keysDirectory)) { - keyParam = commander.keysDirectory - } else { - throw new Error('Missing or invalid private key file/directory') - } + await util.prepareNextVersionCRX( + packagingArgs.binary, + packagingArgs.publisherProofKey, + packagingArgs.endpoint, + packagingArgs.region, + componentClass, + privateKeyFile, + packagingArgs.localRun) } -if (!commander.localRun) { - util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - processJob(commander, keyParam) - }) -} else { - processJob(commander, keyParam) +export { + getPackagingArgs, + packageComponent } diff --git a/scripts/packageEthereumRemoteClient.js b/scripts/packageEthereumRemoteClient.js new file mode 100644 index 00000000..7d076875 --- /dev/null +++ b/scripts/packageEthereumRemoteClient.js @@ -0,0 +1,38 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Example usage: +// npm run package-ethereum-remote-client -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --key-file path/to/ethereum-remote-client.pem + +import path from 'path' +import util from '../lib/util.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' + +const getOriginalManifest = (packageDir) => { + return path.join(packageDir, 'manifest.json') +} + +class EthereumRemoteClient { + constructor () { + const originalManifest = getOriginalManifest(this.packageDir) + const parsedManifest = util.parseManifest(originalManifest) + this.componentId = util.getIDFromBase64PublicKey(parsedManifest.key) + } + + packageDir = path.join('node_modules', 'ethereum-remote-client') + + stagingDir = path.join('build', 'ethereum-remote-client') + crxFile = path.join(this.stagingDir, 'ethereum-remote-client.crx') + + privateKeyFromDir (keyDir) { + return path.join(keyDir, 'ethereum-remote-client.pem') + } + + async stageFiles (version, outputDir) { + util.stageDir(this.packageDir, getOriginalManifest(this.packageDir), version, outputDir) + } +} + +const args = getPackagingArgs() +packageComponent(args, new EthereumRemoteClient()) diff --git a/scripts/packageIpfsDaemon.js b/scripts/packageIpfsDaemon.js index 52971dbb..6a7da056 100644 --- a/scripts/packageIpfsDaemon.js +++ b/scripts/packageIpfsDaemon.js @@ -5,10 +5,9 @@ // Example usage: // npm run package-ipfs-daemon -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --keys-directory path/to/key/dir -import commander from 'commander' -import fs from 'fs' import path from 'path' import util from '../lib/util.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' const ipfsVersion = '0.24.0' const getIpfsDaemonPath = (os, arch) => { @@ -22,63 +21,44 @@ const getOriginalManifest = (platform) => { return path.join('manifests', 'ipfs-daemon-updater', `ipfs-daemon-updater-${platform}-manifest.json`) } -const packageIpfsDaemon = (binary, endpoint, region, os, arch, key, - publisherProofKey) => { - const platform = `${os}-${arch}` - const originalManifest = getOriginalManifest(platform) - const parsedManifest = util.parseManifest(originalManifest) - const id = util.getIDFromBase64PublicKey(parsedManifest.key) +class IpfsDaemon { + constructor (os, arch) { + this.os = os + this.arch = arch + this.platform = `${this.os}-${this.arch}` - util.getNextVersion(endpoint, region, id).then((version) => { - const stagingDir = path.join('build', 'ipfs-daemon-updater', platform) - const ipfsDaemon = getIpfsDaemonPath(os, arch) - const crxOutputDir = path.join('build', 'ipfs-daemon-updater') - const crxFile = path.join(crxOutputDir, `ipfs-daemon-updater-${platform}.crx`) - const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `ipfs-daemon-updater-${platform}.pem`) - stageFiles(platform, ipfsDaemon, version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) -} - -const stageFiles = (platform, ipfsDaemon, version, outputDir) => { - const files = [ - { path: getOriginalManifest(platform), outputName: 'manifest.json' }, - { path: ipfsDaemon } - ] - util.stageFiles(files, version, outputDir) -} + const originalManifest = getOriginalManifest(this.platform) + const parsedManifest = util.parseManifest(originalManifest) + this.componentId = util.getIDFromBase64PublicKey(parsedManifest.key) -util.installErrorHandlers() + this.stagingDir = path.join('build', 'ipfs-daemon-updater', this.platform) + this.crxFile = path.join('build', 'ipfs-daemon-updater', `ipfs-daemon-updater-${this.platform}.crx`) + } -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files', 'abc') - .option('-f, --key-file ', 'private key file for signing crx', 'key.pem')) - .parse(process.argv) + privateKeyFromDir (keyDir) { + return path.join(keyDir, `ipfs-daemon-updater-${this.platform}.pem`) + } -let keyParam = '' - -if (fs.existsSync(commander.keyFile)) { - keyParam = commander.keyFile -} else if (fs.existsSync(commander.keysDirectory)) { - keyParam = commander.keysDirectory -} else { - throw new Error('Missing or invalid private key file/directory') + async stageFiles (version, outputDir) { + const ipfsDaemon = getIpfsDaemonPath(this.os, this.arch) + const files = [ + { path: getOriginalManifest(this.platform), outputName: 'manifest.json' }, + { path: ipfsDaemon } + ] + util.stageFiles(files, version, outputDir) + } } -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, - 'darwin', 'amd64', keyParam, commander.publisherProofKey) - packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, - 'darwin', 'arm64', keyParam, commander.publisherProofKey) - packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, - 'linux', 'amd64', keyParam, commander.publisherProofKey) - packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, - 'linux', 'arm64', keyParam, commander.publisherProofKey) - packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, - 'win32', 'amd64', keyParam, commander.publisherProofKey) - packageIpfsDaemon(commander.binary, commander.endpoint, commander.region, - 'win32', 'arm64', keyParam, commander.publisherProofKey) -}) +const osArchVariants = [ + ['darwin', 'amd64'], + ['darwin', 'arm64'], + ['linux', 'amd64'], + ['linux', 'arm64'], + ['win32', 'amd64'], + ['win32', 'arm64'] +] + +const args = getPackagingArgs() +Promise.all(osArchVariants.map(([os, arch]) => + packageComponent(args, new IpfsDaemon(os, arch)) +)) diff --git a/scripts/packageLocalDataFiles.js b/scripts/packageLocalDataFiles.js index 77dd7c9e..ea864609 100644 --- a/scripts/packageLocalDataFiles.js +++ b/scripts/packageLocalDataFiles.js @@ -5,106 +5,57 @@ // Example usage: // npm run package-local-data-files -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --key-file path/to/local-data-files-updater.pem -import commander from 'commander' -import fs from 'fs-extra' import { mkdirp } from 'mkdirp' import path from 'path' import recursive from 'recursive-readdir-sync' import util from '../lib/util.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' + +const componentType = 'local-data-files-updater' +const datFileName = 'default' const getOriginalManifest = () => { return path.join('manifests', 'local-data-files-updater', 'default-manifest.json') } -const stageFiles = (version, outputDir) => { - const datFileVersion = '1' - const files = [ - { path: getOriginalManifest(), outputName: 'manifest.json' }, - { path: path.join('brave-lists', 'debounce.json'), outputName: path.join(datFileVersion, 'debounce.json') }, - { path: path.join('brave-lists', 'request-otr.json'), outputName: path.join(datFileVersion, 'request-otr.json') }, - { path: path.join('brave-lists', 'clean-urls.json'), outputName: path.join(datFileVersion, 'clean-urls.json') }, - { path: path.join('brave-lists', 'https-upgrade-exceptions-list.txt'), outputName: path.join(datFileVersion, 'https-upgrade-exceptions-list.txt') }, - { path: path.join('brave-lists', 'localhost-permission-allow-list.txt'), outputName: path.join(datFileVersion, 'localhost-permission-allow-list.txt') } - ].concat( - recursive(path.join('node_modules', 'brave-site-specific-scripts', 'dist')).map(f => { - let outputDatDir = datFileVersion - const index = f.indexOf('/dist/') - let baseDir = f.substring(index + '/dist/'.length) - baseDir = baseDir.substring(0, baseDir.lastIndexOf('/')) - outputDatDir = path.join(outputDatDir, baseDir) - mkdirp.sync(path.join(outputDir, outputDatDir)) - return { - path: f, - outputName: path.join(outputDatDir, path.parse(f).base) - } - })) - util.stageFiles(files, version, outputDir) -} - -const postNextVersionWork = (key, publisherProofKey, binary, localRun, version) => { - const componentType = 'local-data-files-updater' - const datFileName = 'default' - const stagingDir = path.join('build', componentType, datFileName) - const crxOutputDir = path.join('build', componentType) - const crxFile = path.join(crxOutputDir, `${componentType}-${datFileName}.crx`) - let privateKeyFile = '' - if (!localRun) { - privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `${componentType}-${datFileName}.pem`) - } - stageFiles(version, stagingDir) - if (!localRun) { - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) +class LocalDataFiles { + stagingDir = path.join('build', componentType, datFileName) + crxFile = path.join('build', componentType, `${componentType}-${datFileName}.crx`) + componentId = (() => { + const originalManifest = getOriginalManifest() + const parsedManifest = util.parseManifest(originalManifest) + return util.getIDFromBase64PublicKey(parsedManifest.key) + })() + + privateKeyFromDir (keyDir) { + return path.join(keyDir, `${componentType}-${datFileName}.pem`) } - console.log(`Generated ${crxFile} with version number ${version}`) -} - -const processDATFile = (binary, endpoint, region, key, publisherProofKey, localRun) => { - const originalManifest = getOriginalManifest() - const parsedManifest = util.parseManifest(originalManifest) - const id = util.getIDFromBase64PublicKey(parsedManifest.key) - if (!localRun) { - util.getNextVersion(endpoint, region, id).then((version) => { - postNextVersionWork(key, publisherProofKey, - binary, localRun, version) - }) - } else { - postNextVersionWork(key, publisherProofKey, - binary, localRun, '1.0.0') + async stageFiles (version, outputDir) { + const datFileVersion = '1' + const files = [ + { path: getOriginalManifest(), outputName: 'manifest.json' }, + { path: path.join('brave-lists', 'debounce.json'), outputName: path.join(datFileVersion, 'debounce.json') }, + { path: path.join('brave-lists', 'request-otr.json'), outputName: path.join(datFileVersion, 'request-otr.json') }, + { path: path.join('brave-lists', 'clean-urls.json'), outputName: path.join(datFileVersion, 'clean-urls.json') }, + { path: path.join('brave-lists', 'https-upgrade-exceptions-list.txt'), outputName: path.join(datFileVersion, 'https-upgrade-exceptions-list.txt') }, + { path: path.join('brave-lists', 'localhost-permission-allow-list.txt'), outputName: path.join(datFileVersion, 'localhost-permission-allow-list.txt') } + ].concat( + recursive(path.join('node_modules', 'brave-site-specific-scripts', 'dist')).map(f => { + let outputDatDir = datFileVersion + const index = f.indexOf('/dist/') + let baseDir = f.substring(index + '/dist/'.length) + baseDir = baseDir.substring(0, baseDir.lastIndexOf('/')) + outputDatDir = path.join(outputDatDir, baseDir) + mkdirp.sync(path.join(outputDir, outputDatDir)) + return { + path: f, + outputName: path.join(outputDatDir, path.parse(f).base) + } + })) + util.stageFiles(files, version, outputDir) } } -const processJob = (commander, keyParam) => { - processDATFile(commander.binary, commander.endpoint, commander.region, - keyParam, commander.publisherProofKey, commander.localRun) -} - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files') - .option('-f, --key-file ', 'private key file for signing crx', 'key.pem') - .option('-l, --local-run', 'Runs updater job without connecting anywhere remotely')) - .parse(process.argv) - -let keyParam = '' - -if (!commander.localRun) { - if (fs.existsSync(commander.keyFile)) { - keyParam = commander.keyFile - } else if (fs.existsSync(commander.keysDirectory)) { - keyParam = commander.keysDirectory - } else { - throw new Error('Missing or invalid private key file/directory') - } -} - -if (!commander.localRun) { - util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - processJob(commander, keyParam) - }) -} else { - processJob(commander, keyParam) -} +const args = getPackagingArgs() +packageComponent(args, new LocalDataFiles()) diff --git a/scripts/packageNTPBackgroundImagesComponent.js b/scripts/packageNTPBackgroundImagesComponent.js index 8839119b..c30f8bc6 100644 --- a/scripts/packageNTPBackgroundImagesComponent.js +++ b/scripts/packageNTPBackgroundImagesComponent.js @@ -2,22 +2,18 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ -import commander from 'commander' import fs from 'fs-extra' -import { mkdirp } from 'mkdirp' import path from 'path' import util from '../lib/util.js' import ntpUtil from '../lib/ntpUtil.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' + +const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-background-images') const getOriginalManifest = () => { return path.join(path.resolve(), 'build', 'ntp-background-images', 'ntp-background-images-manifest.json') } -const stageFiles = util.stageDir.bind( - undefined, - path.join(path.resolve(), 'build', 'ntp-background-images', 'resources'), - getOriginalManifest()) - const generateManifestFile = (publicKey) => { const manifestFile = getOriginalManifest() const manifestContent = { @@ -30,39 +26,28 @@ const generateManifestFile = (publicKey) => { fs.writeFileSync(manifestFile, JSON.stringify(manifestContent)) } -const generateCRXFile = (binary, endpoint, region, componentID, privateKeyFile, - publisherProofKey) => { - const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-background-images') - const stagingDir = path.join(rootBuildDir, 'staging') - const crxOutputDir = path.join(rootBuildDir, 'output') - mkdirp.sync(stagingDir) - mkdirp.sync(crxOutputDir) - util.getNextVersion(endpoint, region, componentID).then((version) => { - const crxFile = path.join(crxOutputDir, 'ntp-background-images.crx') - stageFiles(version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) -} - -util.installErrorHandlers() +class NTPBackgroundImagesComponent { + constructor (privateKeyFile) { + const [publicKey, componentId] = ntpUtil.generatePublicKeyAndID(privateKeyFile) + this.publicKey = publicKey + this.componentId = componentId + } -util.addCommonScriptOptions( - commander - .option('-k, --key ', 'file containing private key for signing crx file')) - .parse(process.argv) + stagingDir = path.join(rootBuildDir, 'staging') + crxFile = path.join(rootBuildDir, 'output', 'ntp-background-images.crx') -let privateKeyFile = '' -if (fs.existsSync(commander.key)) { - privateKeyFile = commander.key -} else { - throw new Error('Missing or invalid private key') + async stageFiles (version, outputDir) { + generateManifestFile(this.publicKey) + util.stageDir( + path.join(path.resolve(), 'build', 'ntp-background-images', 'resources'), + getOriginalManifest(), + version, + outputDir) + } } -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - const [publicKey, componentID] = ntpUtil.generatePublicKeyAndID(privateKeyFile) - generateManifestFile(publicKey) - generateCRXFile(commander.binary, commander.endpoint, commander.region, - componentID, privateKeyFile, commander.publisherProofKey) -}) +const args = getPackagingArgs() +if (args.keyFile === undefined) { + throw new Error('--key-file is required') +} +packageComponent(args, new NTPBackgroundImagesComponent(args.keyFile)) diff --git a/scripts/packageNTPSuperReferrerComponent.js b/scripts/packageNTPSuperReferrerComponent.js index 7d4cb7df..c9f14ef9 100644 --- a/scripts/packageNTPSuperReferrerComponent.js +++ b/scripts/packageNTPSuperReferrerComponent.js @@ -2,20 +2,13 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ -import commander from 'commander' import fs from 'fs-extra' -import { mkdirp } from 'mkdirp' import path from 'path' import util from '../lib/util.js' import ntpUtil from '../lib/ntpUtil.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' -const stageFiles = (superReferrerName, version, outputDir) => { - util.stageDir( - path.join(path.resolve(), 'build', 'ntp-super-referrer', 'resources', superReferrerName, '/'), - getOriginalManifest(superReferrerName), - version, - outputDir) -} +const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer') const generateManifestFile = (superReferrerName, publicKey) => { const manifestFile = getOriginalManifest(superReferrerName) @@ -33,41 +26,30 @@ const getOriginalManifest = (superReferrerName) => { return path.join(path.resolve(), 'build', 'ntp-super-referrer', `${superReferrerName}-manifest.json`) } -const generateCRXFile = (binary, endpoint, region, superReferrerName, - componentID, privateKeyFile, publisherProofKey) => { - const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer') - const stagingDir = path.join(rootBuildDir, 'staging', superReferrerName) - const crxOutputDir = path.join(rootBuildDir, 'output') - mkdirp.sync(stagingDir) - mkdirp.sync(crxOutputDir) - util.getNextVersion(endpoint, region, componentID).then((version) => { - const crxFile = path.join(crxOutputDir, `ntp-super-referrer-${superReferrerName}.crx`) - stageFiles(superReferrerName, version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) -} - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-n, --super-referrer-name ', 'super referrer name for this component') - .option('-k, --key ', 'file containing private key for signing crx file')) - .parse(process.argv) +class NTPSuperReferrerComponent { + constructor (superReferrerName, privateKeyFile) { + const [publicKey, componentId] = ntpUtil.generatePublicKeyAndID(privateKeyFile) + this.publicKey = publicKey + this.componentId = componentId + this.superReferrerName = superReferrerName + this.stagingDir = path.join(rootBuildDir, 'staging', this.superReferrerName) + this.crxFile = path.join(rootBuildDir, 'output', `ntp-super-referrer-${this.superReferrerName}.crx`) + } -let privateKeyFile = '' -if (fs.existsSync(commander.key)) { - privateKeyFile = commander.key -} else { - throw new Error('Missing or invalid private key') + async stageFiles (version, outputDir) { + generateManifestFile(this.superReferrerName, this.publicKey) + util.stageDir( + path.join(path.resolve(), 'build', 'ntp-super-referrer', 'resources', this.superReferrerName, '/'), + getOriginalManifest(this.superReferrerName), + version, + outputDir) + } } -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - const [publicKey, componentID] = ntpUtil.generatePublicKeyAndID(privateKeyFile) - generateManifestFile(commander.superReferrerName, publicKey) - generateCRXFile(commander.binary, commander.endpoint, commander.region, - commander.superReferrerName, componentID, privateKeyFile, - commander.publisherProofKey) -}) +const args = getPackagingArgs([ + ['-n, --super-referrer-name ', 'super referrer name for this component'] +]) +if (args.keyFile === undefined) { + throw new Error('--key-file is required') +} +packageComponent(args, new NTPSuperReferrerComponent(args.superReferrerName, args.keyFile)) diff --git a/scripts/packageNTPSuperReferrerMappingTableComponent.js b/scripts/packageNTPSuperReferrerMappingTableComponent.js index 499db25c..ac2d57a8 100644 --- a/scripts/packageNTPSuperReferrerMappingTableComponent.js +++ b/scripts/packageNTPSuperReferrerMappingTableComponent.js @@ -2,22 +2,19 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ -import commander from 'commander' import fs from 'fs-extra' -import { mkdirp } from 'mkdirp' import path from 'path' import util from '../lib/util.js' import ntpUtil from '../lib/ntpUtil.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' -const stageFiles = (version, outputDir) => { - const files = [ - { path: getOriginalManifest(), outputName: 'manifest.json' }, - { path: path.join(path.resolve(), 'build', 'ntp-super-referrer', 'resources', 'mapping-table', 'mapping-table.json') } - ] - util.stageFiles(files, version, outputDir) +const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer', 'mapping-table') + +const getOriginalManifest = () => { + return path.join(path.resolve(), 'build', 'ntp-super-referrer', 'mapping-table-manifest.json') } -const generateManifestFile = (publicKey) => { +const generateManifestFile = async (publicKey) => { const manifestFile = getOriginalManifest() const manifestContent = { description: 'Brave NTP Super Referrer mapping table component', @@ -29,43 +26,28 @@ const generateManifestFile = (publicKey) => { fs.writeFileSync(manifestFile, JSON.stringify(manifestContent)) } -const getOriginalManifest = () => { - return path.join(path.resolve(), 'build', 'ntp-super-referrer', 'mapping-table-manifest.json') -} - -const generateCRXFile = (binary, endpoint, region, componentID, privateKeyFile, - publisherProofKey) => { - const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer', 'mapping-table') - const stagingDir = path.join(rootBuildDir, 'staging') - const crxOutputDir = path.join(rootBuildDir, 'output') - mkdirp.sync(stagingDir) - mkdirp.sync(crxOutputDir) - util.getNextVersion(endpoint, region, componentID).then((version) => { - const crxFile = path.join(crxOutputDir, 'ntp-super-referrer-mapping-table.crx') - stageFiles(version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) -} - -util.installErrorHandlers() +class NTPSuperReferrerMappingTableComponent { + constructor (privateKeyFile) { + const [publicKey, componentId] = ntpUtil.generatePublicKeyAndID(privateKeyFile) + this.publicKey = publicKey + this.componentId = componentId + } -util.addCommonScriptOptions( - commander - .option('-k, --key ', 'file containing private key for signing crx file')) - .parse(process.argv) + stagingDir = path.join(rootBuildDir, 'staging') + crxFile = path.join(rootBuildDir, 'output', 'ntp-super-referrer-mapping-table.crx') -let privateKeyFile = '' -if (fs.existsSync(commander.key)) { - privateKeyFile = commander.key -} else { - throw new Error('Missing or invalid private key') + async stageFiles (version, outputDir) { + generateManifestFile(this.publicKey) + const files = [ + { path: getOriginalManifest(), outputName: 'manifest.json' }, + { path: path.join(path.resolve(), 'build', 'ntp-super-referrer', 'resources', 'mapping-table', 'mapping-table.json') } + ] + util.stageFiles(files, version, outputDir) + } } -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - const [publicKey, componentID] = ntpUtil.generatePublicKeyAndID(privateKeyFile) - generateManifestFile(publicKey) - generateCRXFile(commander.binary, commander.endpoint, commander.region, - componentID, privateKeyFile, commander.publisherProofKey) -}) +const args = getPackagingArgs() +if (args.keyFile === undefined) { + throw new Error('--key-file is required') +} +packageComponent(args, new NTPSuperReferrerMappingTableComponent(args.keyFile)) diff --git a/scripts/packagePlaylist.js b/scripts/packagePlaylist.js index 1654184c..c8856e11 100644 --- a/scripts/packagePlaylist.js +++ b/scripts/packagePlaylist.js @@ -2,54 +2,34 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ -import commander from 'commander' -import fs from 'fs-extra' -import { mkdirp } from 'mkdirp' import path from 'path' import util from '../lib/util.js' import ntpUtil from '../lib/ntpUtil.js' - -const getOriginalManifest = () => { - return path.join(path.resolve(), 'node_modules', 'playlist-component', 'manifest.json') +import { getPackagingArgs, packageComponent } from './packageComponent.js' + +const rootBuildDir = path.join(path.resolve(), 'build', 'playlist') + +class Playlist { + constructor (privateKeyFile) { + const [, componentId] = ntpUtil.generatePublicKeyAndID(privateKeyFile) + this.componentId = componentId + } + + stagingDir = path.join(rootBuildDir, 'staging') + crxFile = path.join(rootBuildDir, 'output', 'playlist.crx') + + async stageFiles (version, outputDir) { + const originalManifest = path.join(path.resolve(), 'node_modules', 'playlist-component', 'manifest.json') + util.stageDir( + path.join(path.resolve(), 'node_modules', 'playlist-component'), + originalManifest, + version, + outputDir) + } } -const stageFiles = util.stageDir.bind( - undefined, - path.join(path.resolve(), 'node_modules', 'playlist-component'), - getOriginalManifest()) - -const generateCRXFile = (binary, endpoint, region, componentID, privateKeyFile, - publisherProofKey) => { - const rootBuildDir = path.join(path.resolve(), 'build', 'playlist') - const stagingDir = path.join(rootBuildDir, 'staging') - const crxOutputDir = path.join(rootBuildDir, 'output') - mkdirp.sync(stagingDir) - mkdirp.sync(crxOutputDir) - util.getNextVersion(endpoint, region, componentID).then((version) => { - const crxFile = path.join(crxOutputDir, 'playlist.crx') - stageFiles(version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) +const args = getPackagingArgs() +if (args.keyFile === undefined) { + throw new Error('--key-file is required') } - -util.installErrorHandlers() - -util.addCommonScriptOptions( - commander - .option('-k, --key ', 'file containing private key for signing crx file')) - .parse(process.argv) - -let privateKeyFile = '' -if (fs.existsSync(commander.key)) { - privateKeyFile = commander.key -} else { - throw new Error('Missing or invalid private key') -} - -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - const [, componentID] = ntpUtil.generatePublicKeyAndID(privateKeyFile) - generateCRXFile(commander.binary, commander.endpoint, commander.region, - componentID, privateKeyFile, commander.publisherProofKey) -}) +packageComponent(args, new Playlist(args.keyFile)) diff --git a/scripts/packageTorClient.js b/scripts/packageTorClient.js index 99bf279f..cb1b28aa 100644 --- a/scripts/packageTorClient.js +++ b/scripts/packageTorClient.js @@ -5,13 +5,13 @@ // Example usage: // npm run package-tor-client -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --keys-directory path/to/key/dir -import commander from 'commander' import crypto from 'crypto' import { execSync } from 'child_process' import fs from 'fs' import { mkdirp } from 'mkdirp' import path from 'path' import util from '../lib/util.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' // Downloads the current (platform-specific) Tor client from S3 const downloadTorClient = (platform) => { @@ -63,38 +63,6 @@ const downloadTorClient = (platform) => { return torClient } -const getOriginalManifest = (platform) => { - return path.join('manifests', 'tor-client-updater', `tor-client-updater-${platform}-manifest.json`) -} - -const packageTorClient = (binary, endpoint, region, platform, key, - publisherProofKey) => { - const originalManifest = getOriginalManifest(platform) - const parsedManifest = util.parseManifest(originalManifest) - const id = util.getIDFromBase64PublicKey(parsedManifest.key) - - util.getNextVersion(endpoint, region, id).then((version) => { - const stagingDir = path.join('build', 'tor-client-updater', platform) - const torClient = downloadTorClient(platform) - const crxOutputDir = path.join('build', 'tor-client-updater') - const crxFile = path.join(crxOutputDir, `tor-client-updater-${platform}.crx`) - const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `tor-client-updater-${platform}.pem`) - stageFiles(platform, torClient, version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, - stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) -} - -const stageFiles = (platform, torClient, version, outputDir) => { - const files = [ - { path: getOriginalManifest(platform), outputName: 'manifest.json' }, - { path: torClient }, - { path: path.join('resources', 'tor', 'torrc'), outputName: 'tor-torrc' } - ] - util.stageFiles(files, version, outputDir) -} - // Does a hash comparison on a file against a given hash const verifyChecksum = (file, hash) => { const filecontent = fs.readFileSync(file) @@ -103,31 +71,37 @@ const verifyChecksum = (file, hash) => { return hash === computedHash } -util.installErrorHandlers() +const getOriginalManifest = (platform) => { + return path.join('manifests', 'tor-client-updater', `tor-client-updater-${platform}-manifest.json`) +} + +class TorClient { + constructor (platform) { + this.platform = platform + const originalManifest = getOriginalManifest(this.platform) + const parsedManifest = util.parseManifest(originalManifest) + this.componentId = util.getIDFromBase64PublicKey(parsedManifest.key) -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files', 'abc') - .option('-f, --key-file ', 'private key file for signing crx', 'key.pem')) - .parse(process.argv) + this.stagingDir = path.join('build', 'tor-client-updater', this.platform) + this.crxFile = path.join('build', 'tor-client-updater', `tor-client-updater-${this.platform}.crx`) + } -let keyParam = '' + privateKeyFromDir (keyDir) { + return path.join(keyDir, `tor-client-updater-${this.platform}.pem`) + } -if (fs.existsSync(commander.keyFile)) { - keyParam = commander.keyFile -} else if (fs.existsSync(commander.keysDirectory)) { - keyParam = commander.keysDirectory -} else { - throw new Error('Missing or invalid private key file/directory') + async stageFiles (version, outputDir) { + const torClient = downloadTorClient(this.platform) + const files = [ + { path: getOriginalManifest(this.platform), outputName: 'manifest.json' }, + { path: torClient }, + { path: path.join('resources', 'tor', 'torrc'), outputName: 'tor-torrc' } + ] + util.stageFiles(files, version, outputDir) + } } -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - packageTorClient(commander.binary, commander.endpoint, commander.region, - 'darwin', keyParam, commander.publisherProofKey) - packageTorClient(commander.binary, commander.endpoint, commander.region, - 'linux', keyParam, commander.publisherProofKey) - packageTorClient(commander.binary, commander.endpoint, commander.region, - 'linux-arm64', keyParam, commander.publisherProofKey) - packageTorClient(commander.binary, commander.endpoint, commander.region, - 'win32', keyParam, commander.publisherProofKey) -}) +const platforms = ['darwin', 'linux', 'linux-arm64', 'win32'] + +const args = getPackagingArgs() +Promise.all(platforms.map(platform => packageComponent(args, new TorClient(platform)))) diff --git a/scripts/packageTorPluggableTransports.js b/scripts/packageTorPluggableTransports.js index 11cef25c..66d1b70d 100644 --- a/scripts/packageTorPluggableTransports.js +++ b/scripts/packageTorPluggableTransports.js @@ -5,12 +5,12 @@ // Example usage: // npm run package-tor-pluggable-transports -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --keys-directory path/to/key/dir -import commander from 'commander' import { execSync } from 'child_process' import fs from 'fs' import { mkdirp } from 'mkdirp' import path from 'path' import util from '../lib/util.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' const TOR_PLUGGABLE_TRANSPORTS_UPDATER = 'tor-pluggable-transports-updater' @@ -45,57 +45,36 @@ const getOriginalManifest = (platform) => { return path.join('manifests', TOR_PLUGGABLE_TRANSPORTS_UPDATER, `${TOR_PLUGGABLE_TRANSPORTS_UPDATER}-${platform}-manifest.json`) } -const packageTorPluggableTransports = (binary, endpoint, region, platform, key, publisherProofKey) => { - const originalManifest = getOriginalManifest(platform) - const parsedManifest = util.parseManifest(originalManifest) - const id = util.getIDFromBase64PublicKey(parsedManifest.key) - - util.getNextVersion(endpoint, region, id).then((version) => { - const snowflake = downloadTorPluggableTransport(platform, 'snowflake') - const obfs4 = downloadTorPluggableTransport(platform, 'obfs4') - - const stagingDir = path.join('build', TOR_PLUGGABLE_TRANSPORTS_UPDATER, platform) - const crxOutputDir = path.join('build', TOR_PLUGGABLE_TRANSPORTS_UPDATER) - const crxFile = path.join(crxOutputDir, `${TOR_PLUGGABLE_TRANSPORTS_UPDATER}-${platform}.crx`) - const privateKeyFile = !fs.lstatSync(key).isDirectory() ? key : path.join(key, `${TOR_PLUGGABLE_TRANSPORTS_UPDATER}-${platform}.pem`) - stageFiles(platform, snowflake, obfs4, version, stagingDir) - util.generateCRXFile(binary, crxFile, privateKeyFile, publisherProofKey, stagingDir) - console.log(`Generated ${crxFile} with version number ${version}`) - }) -} +class TorPluggableTransports { + constructor (platform) { + this.platform = platform -const stageFiles = (platform, snowflake, obfs4, version, outputDir) => { - const files = [ - { path: getOriginalManifest(platform), outputName: 'manifest.json' }, - { path: snowflake }, - { path: obfs4 } - ] - util.stageFiles(files, version, outputDir) -} + const originalManifest = getOriginalManifest(this.platform) + const parsedManifest = util.parseManifest(originalManifest) + this.componentId = util.getIDFromBase64PublicKey(parsedManifest.key) -util.installErrorHandlers() + this.stagingDir = path.join('build', TOR_PLUGGABLE_TRANSPORTS_UPDATER, this.platform) + this.crxFile = path.join('build', TOR_PLUGGABLE_TRANSPORTS_UPDATER, `${TOR_PLUGGABLE_TRANSPORTS_UPDATER}-${this.platform}.crx`) + } -util.addCommonScriptOptions( - commander - .option('-d, --keys-directory ', 'directory containing private keys for signing crx files', 'abc') - .option('-f, --key-file ', 'private key file for signing crx', 'key.pem')) - .parse(process.argv) + privateKeyFromDir (keyDir) { + return path.join(keyDir, `${TOR_PLUGGABLE_TRANSPORTS_UPDATER}-${this.platform}.pem`) + } -let keyParam = '' + async stageFiles (version, outputDir) { + const snowflake = downloadTorPluggableTransport(this.platform, 'snowflake') + const obfs4 = downloadTorPluggableTransport(this.platform, 'obfs4') -if (fs.existsSync(commander.keyFile)) { - keyParam = commander.keyFile -} else if (fs.existsSync(commander.keysDirectory)) { - keyParam = commander.keysDirectory -} else { - throw new Error('Missing or invalid private key file/directory') + const files = [ + { path: getOriginalManifest(this.platform), outputName: 'manifest.json' }, + { path: snowflake }, + { path: obfs4 } + ] + util.stageFiles(files, version, outputDir) + } } -util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { - packageTorPluggableTransports(commander.binary, commander.endpoint, commander.region, - 'darwin', keyParam, commander.publisherProofKey) - packageTorPluggableTransports(commander.binary, commander.endpoint, commander.region, - 'linux', keyParam, commander.publisherProofKey) - packageTorPluggableTransports(commander.binary, commander.endpoint, commander.region, - 'win32', keyParam, commander.publisherProofKey) -}) +const platforms = ['darwin', 'linux', 'win32'] + +const args = getPackagingArgs() +Promise.all(platforms.map(platform => packageComponent(args, new TorPluggableTransports(platform)))) diff --git a/scripts/packageWalletDataFiles.js b/scripts/packageWalletDataFiles.js new file mode 100644 index 00000000..7755fc3f --- /dev/null +++ b/scripts/packageWalletDataFiles.js @@ -0,0 +1,40 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Example usage: +// npm run package-wallet-data-files -- --binary "/Applications/Google\\ Chrome\\ Canary.app/Contents/MacOS/Google\\ Chrome\\ Canary" --key-file path/to/wallet-data-files-updater.pem + +import fs from 'fs-extra' +import path from 'path' +import util from '../lib/util.js' +import { getPackagingArgs, packageComponent } from './packageComponent.js' + +const getOriginalManifest = (packageDir) => { + return path.join(packageDir, 'manifest.json') +} + +class WalletDataFilesUpdater { + constructor () { + const originalManifest = getOriginalManifest(this.packageDir) + const parsedManifest = util.parseManifest(originalManifest) + this.componentId = util.getIDFromBase64PublicKey(parsedManifest.key) + } + + packageDir = path.join('node_modules', 'brave-wallet-lists') + + stagingDir = path.join('build', 'wallet-data-files-updater') + crxFile = path.join(this.stagingDir, 'wallet-data-files-updater.crx') + + privateKeyFromDir (keyDir) { + return path.join(keyDir, 'wallet-data-files-updater.pem') + } + + async stageFiles (version, outputDir) { + util.stageDir(this.packageDir, getOriginalManifest(this.packageDir), version, outputDir) + fs.unlinkSync(path.join(outputDir, 'package.json')) + } +} + +const args = getPackagingArgs() +packageComponent(args, new WalletDataFilesUpdater())