Refactor package dependencies generators (microsoft#157845)

Author: rzhao271

build/gulpfile.vscode.linux.js

@@ -15,8 +15,7 @@ const util = require('./lib/util');
 const task = require('./lib/task');
 const packageJson = require('../package.json');
 const product = require('../product.json');
-const rpmDependenciesGenerator = require('./linux/rpm/dependencies-generator');
-const debianDependenciesGenerator = require('./linux/debian/dependencies-generator');
+const dependenciesGenerator = require('./linux/dependencies-generator');
 const sysrootInstaller = require('./linux/debian/install-sysroot');
 const debianRecommendedDependencies = require('./linux/debian/dep-lists').recommendedDeps;
 const path = require('path');
@@ -25,6 +24,9 @@ const commit = util.getVersion(root);
 
 const linuxPackageRevision = Math.floor(new Date().getTime() / 1000);
 
+/**
+ * @param {string} arch
+ */
 function getDebPackageArch(arch) {
 	return { x64: 'amd64', armhf: 'armhf', arm64: 'arm64' }[arch];
 }
@@ -80,7 +82,7 @@ function prepareDebPackage(arch) {
 			async function () {
 				const that = this;
 				const sysroot = await sysrootInstaller.getSysroot(debArch);
-				const dependencies = debianDependenciesGenerator.getDependencies(binaryDir, product.applicationName, debArch, sysroot);
+				const dependencies = dependenciesGenerator.getDependencies('deb', binaryDir, product.applicationName, debArch, sysroot);
 				gulp.src('resources/linux/debian/control.template', { base: '.' })
 					.pipe(replace('@@NAME@@', product.applicationName))
 					.pipe(replace('@@VERSION@@', packageJson.version + '-' + linuxPackageRevision))
@@ -183,7 +185,7 @@ function prepareRpmPackage(arch) {
 		const code = gulp.src(binaryDir + '/**/*', { base: binaryDir })
 			.pipe(rename(function (p) { p.dirname = 'BUILD/usr/share/' + product.applicationName + '/' + p.dirname; }));
 
-		const dependencies = rpmDependenciesGenerator.getDependencies(binaryDir, product.applicationName, rpmArch);
+		const dependencies = dependenciesGenerator.getDependencies('rpm', binaryDir, product.applicationName, rpmArch);
 		const spec = gulp.src('resources/linux/rpm/code.spec.template', { base: '.' })
 			.pipe(replace('@@NAME@@', product.applicationName))
 			.pipe(replace('@@NAME_LONG@@', product.nameLong))

build/linux/debian/dependencies-generator.js renamed to build/linux/debian/calculate-deps.js

@@ -1,70 +1,23 @@
+"use strict";
 /*---------------------------------------------------------------------------------------------
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-'use strict';
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getDependencies = void 0;
+exports.generatePackageDeps = void 0;
 const child_process_1 = require("child_process");
 const fs_1 = require("fs");
 const os_1 = require("os");
 const path = require("path");
-const dep_lists_1 = require("./dep-lists");
 const manifests = require("../../../cgmanifest.json");
-// A flag that can easily be toggled.
-// Make sure to compile the build directory after toggling the value.
-// If false, we warn about new dependencies if they show up
-// while running the Debian prepare package task for a release.
-// If true, we fail the build if there are new dependencies found during that task.
-// The reference dependencies, which one has to update when the new dependencies
-// are valid, are in dep-lists.ts
-const FAIL_BUILD_FOR_NEW_DEPENDENCIES = true;
-function getDependencies(buildDir, applicationName, arch, sysroot) {
-    // Get the files for which we want to find dependencies.
-    const nativeModulesPath = path.join(buildDir, 'resources', 'app', 'node_modules.asar.unpacked');
-    const findResult = (0, child_process_1.spawnSync)('find', [nativeModulesPath, '-name', '*.node']);
-    if (findResult.status) {
-        console.error('Error finding files:');
-        console.error(findResult.stderr.toString());
-        return [];
-    }
-    const files = findResult.stdout.toString().trimEnd().split('\n');
-    const appPath = path.join(buildDir, applicationName);
-    files.push(appPath);
-    // Add chrome sandbox and crashpad handler.
-    files.push(path.join(buildDir, 'chrome-sandbox'));
-    files.push(path.join(buildDir, 'chrome_crashpad_handler'));
-    // Generate the dependencies.
-    const dependencies = files.map((file) => calculatePackageDeps(file, arch, sysroot));
-    // Add additional dependencies.
+const dep_lists_1 = require("./dep-lists");
+function generatePackageDeps(files, arch, sysroot) {
+    const dependencies = files.map(file => calculatePackageDeps(file, arch, sysroot));
     const additionalDepsSet = new Set(dep_lists_1.additionalDeps);
     dependencies.push(additionalDepsSet);
-    // Merge all the dependencies.
-    const mergedDependencies = mergePackageDeps(dependencies);
-    let sortedDependencies = [];
-    for (const dependency of mergedDependencies) {
-        sortedDependencies.push(dependency);
-    }
-    sortedDependencies.sort();
-    // Exclude bundled dependencies
-    sortedDependencies = sortedDependencies.filter(dependency => {
-        return !dep_lists_1.bundledDeps.some(bundledDep => dependency.startsWith(bundledDep));
-    });
-    const referenceGeneratedDeps = dep_lists_1.referenceGeneratedDepsByArch[arch];
-    if (JSON.stringify(sortedDependencies) !== JSON.stringify(referenceGeneratedDeps)) {
-        const failMessage = 'The dependencies list has changed.'
-            + '\nOld:\n' + referenceGeneratedDeps.join('\n')
-            + '\nNew:\n' + sortedDependencies.join('\n');
-        if (FAIL_BUILD_FOR_NEW_DEPENDENCIES) {
-            throw new Error(failMessage);
-        }
-        else {
-            console.warn(failMessage);
-        }
-    }
-    return sortedDependencies;
+    return dependencies;
 }
-exports.getDependencies = getDependencies;
+exports.generatePackageDeps = generatePackageDeps;
 // Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/debian/calculate_package_deps.py.
 function calculatePackageDeps(binaryPath, arch, sysroot) {
     try {
@@ -97,8 +50,6 @@ function calculatePackageDeps(binaryPath, arch, sysroot) {
         case 'arm64':
             cmd.push(`-l${sysroot}/usr/lib/aarch64-linux-gnu`, `-l${sysroot}/lib/aarch64-linux-gnu`);
             break;
-        default:
-            throw new Error('Unsupported architecture ' + arch);
     }
     cmd.push(`-l${sysroot}/usr/lib`);
     cmd.push('-O', '-e', path.resolve(binaryPath));
@@ -117,17 +68,3 @@ function calculatePackageDeps(binaryPath, arch, sysroot) {
     const requires = new Set(depsStr.split(', ').sort());
     return requires;
 }
-// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/rpm/merge_package_deps.py.
-function mergePackageDeps(inputDeps) {
-    // For now, see if directly appending the dependencies helps.
-    const requires = new Set();
-    for (const depSet of inputDeps) {
-        for (const dep of depSet) {
-            const trimmedDependency = dep.trim();
-            if (trimmedDependency.length && !trimmedDependency.startsWith('#')) {
-                requires.add(trimmedDependency);
-            }
-        }
-    }
-    return requires;
-}

build/linux/debian/calculate-deps.ts

@@ -0,0 +1,75 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { spawnSync } from 'child_process';
+import { constants, statSync } from 'fs';
+import { tmpdir } from 'os';
+import path = require('path');
+import * as manifests from '../../../cgmanifest.json';
+import { additionalDeps } from './dep-lists';
+import { DebianArchString } from './types';
+
+export function generatePackageDeps(files: string[], arch: DebianArchString, sysroot: string): Set<string>[] {
+	const dependencies: Set<string>[] = files.map(file => calculatePackageDeps(file, arch, sysroot));
+	const additionalDepsSet = new Set(additionalDeps);
+	dependencies.push(additionalDepsSet);
+	return dependencies;
+}
+
+// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/debian/calculate_package_deps.py.
+function calculatePackageDeps(binaryPath: string, arch: DebianArchString, sysroot: string): Set<string> {
+	try {
+		if (!(statSync(binaryPath).mode & constants.S_IXUSR)) {
+			throw new Error(`Binary ${binaryPath} needs to have an executable bit set.`);
+		}
+	} catch (e) {
+		// The package might not exist. Don't re-throw the error here.
+		console.error('Tried to stat ' + binaryPath + ' but failed.');
+	}
+
+	// Get the Chromium dpkg-shlibdeps file.
+	const chromiumManifest = manifests.registrations.filter(registration => {
+		return registration.component.type === 'git' && registration.component.git!.name === 'chromium';
+	});
+	const dpkgShlibdepsUrl = `https://raw.githubusercontent.com/chromium/chromium/${chromiumManifest[0].version}/third_party/dpkg-shlibdeps/dpkg-shlibdeps.pl`;
+	const dpkgShlibdepsScriptLocation = `${tmpdir()}/dpkg-shlibdeps.pl`;
+	const result = spawnSync('curl', [dpkgShlibdepsUrl, '-o', dpkgShlibdepsScriptLocation]);
+	if (result.status !== 0) {
+		throw new Error('Cannot retrieve dpkg-shlibdeps. Stderr:\n' + result.stderr);
+	}
+	const cmd = [dpkgShlibdepsScriptLocation, '--ignore-weak-undefined'];
+	switch (arch) {
+		case 'amd64':
+			cmd.push(`-l${sysroot}/usr/lib/x86_64-linux-gnu`,
+				`-l${sysroot}/lib/x86_64-linux-gnu`);
+			break;
+		case 'armhf':
+			cmd.push(`-l${sysroot}/usr/lib/arm-linux-gnueabihf`,
+				`-l${sysroot}/lib/arm-linux-gnueabihf`);
+			break;
+		case 'arm64':
+			cmd.push(`-l${sysroot}/usr/lib/aarch64-linux-gnu`,
+				`-l${sysroot}/lib/aarch64-linux-gnu`);
+			break;
+	}
+	cmd.push(`-l${sysroot}/usr/lib`);
+	cmd.push('-O', '-e', path.resolve(binaryPath));
+
+	const dpkgShlibdepsResult = spawnSync('perl', cmd, { cwd: sysroot });
+	if (dpkgShlibdepsResult.status !== 0) {
+		throw new Error(`dpkg-shlibdeps failed with exit code ${dpkgShlibdepsResult.status}. stderr:\n${dpkgShlibdepsResult.stderr} `);
+	}
+
+	const shlibsDependsPrefix = 'shlibs:Depends=';
+	const requiresList = dpkgShlibdepsResult.stdout.toString('utf-8').trimEnd().split('\n');
+	let depsStr = '';
+	for (const line of requiresList) {
+		if (line.startsWith(shlibsDependsPrefix)) {
+			depsStr = line.substring(shlibsDependsPrefix.length);
+		}
+	}
+	const requires = new Set(depsStr.split(', ').sort());
+	return requires;
+}

build/linux/debian/dep-lists.js

@@ -4,7 +4,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.referenceGeneratedDepsByArch = exports.bundledDeps = exports.recommendedDeps = exports.additionalDeps = void 0;
+exports.referenceGeneratedDepsByArch = exports.recommendedDeps = exports.additionalDeps = void 0;
 // Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/debian/additional_deps
 // Additional dependencies not in the dpkg-shlibdeps output.
 exports.additionalDeps = [
@@ -20,18 +20,6 @@ exports.additionalDeps = [
 exports.recommendedDeps = [
     'libvulkan1' // Move to additionalDeps once support for Trusty and Jessie are dropped.
 ];
-// Based on https://source.chromium.org/chromium/chromium/src/+/refs/tags/98.0.4758.109:chrome/installer/linux/BUILD.gn;l=64-80
-// and the Linux Archive build
-// Shared library dependencies that we already bundle.
-exports.bundledDeps = [
-    'libEGL.so',
-    'libGLESv2.so',
-    'libvulkan.so.1',
-    'swiftshader_libEGL.so',
-    'swiftshader_libGLESv2.so',
-    'libvk_swiftshader.so',
-    'libffmpeg.so'
-];
 exports.referenceGeneratedDepsByArch = {
     'amd64': [
         'ca-certificates',

build/linux/debian/dep-lists.ts

@@ -20,19 +20,6 @@ export const recommendedDeps = [
 	'libvulkan1' // Move to additionalDeps once support for Trusty and Jessie are dropped.
 ];
 
-// Based on https://source.chromium.org/chromium/chromium/src/+/refs/tags/98.0.4758.109:chrome/installer/linux/BUILD.gn;l=64-80
-// and the Linux Archive build
-// Shared library dependencies that we already bundle.
-export const bundledDeps = [
-	'libEGL.so',
-	'libGLESv2.so',
-	'libvulkan.so.1',
-	'swiftshader_libEGL.so',
-	'swiftshader_libGLESv2.so',
-	'libvk_swiftshader.so',
-	'libffmpeg.so'
-];
-
 export const referenceGeneratedDepsByArch = {
 	'amd64': [
 		'ca-certificates',