feat(generate): Add support for generating bundles with dev installs (#62)

* Add npm util to find all packages inside a directory

* Add support for generating bundles with dev installs

* Fix tests

* Clean code

* Test whether bundles generated with dev installs fetch the packages using tarballs

* Update readme

Author: hlxid

README.md

@@ -48,7 +48,8 @@ Uses your installed nodecg-io version and services, meaning you need to have the
 These generated bundles are only meant as a starting point, you may probably do more things like creating a git repository for your bundle,
 add a licence, or add other tools like linters.
 
-Also, this command currently only works with installs of released versions and not with development installs. This is because all bundles using nodecg-io depend on `nodecg-io-core` and if you use typescript each used service as well. For development installs these are not published on npm, and you would need some way of linking the packages locally.
+If you are using a released version of nodecg-io (aka. a production install) the nodecg-io packages get fetched directly from npm.
+If you are using a development version of nodecg-io these get fetched as tarballs from the [nodecg-io-publish repository](https://github.com/codeoverflow-org/nodecg-io-publish).
 
 ## A note about versioning
 
@@ -59,9 +60,7 @@ The following table show which versions of the CLI are compatible with which nod
 | CLI versions | nodecg-io versions |
 | ------------ | ------------------ |
 | `0.1`        | `0.1`              |
-| `0.2`        | `0.2`, `0.1`       |
-
-Currently, they are the same, but we will follow [semver2](https://semver.org/) using [semantic-release](https://semantic-release.gitbook.io/semantic-release/) and the versions will diverge at some point.
+| `0.3`, `0.2` | `0.2`, `0.1`       |
 
 ## Developer workflow
 

src/generate/extension.ts

@@ -1,6 +1,6 @@
 import CodeBlockWriter from "code-block-writer";
 import { getServiceClientName } from "../nodecgIOVersions";
-import { ProductionInstallation } from "../utils/installation";
+import { Installation } from "../utils/installation";
 import { CodeLanguage, GenerationOptions } from "./prompt";
 import { writeBundleFile } from "./utils";
 
@@ -33,12 +33,15 @@ function getServiceNames(serviceBaseName: string, nodecgIOVersion: string): Serv
     };
 }
 
-export async function genExtension(opts: GenerationOptions, install: ProductionInstallation): Promise<void> {
-    // Generate further information for each service which is needed to generate the bundle extension.
-    const services = opts.services.map((svc) => getServiceNames(svc, install.version));
+export async function genExtension(opts: GenerationOptions, install: Installation): Promise<void> {
+    // Generate all variants of the service names if were doing it from a production install.
+    // We can't generate the imports and stuff if we currently have a development install because
+    // the service names for each version are hardcoded and unknown for a development version.
+    const services = install.dev === false ? opts.services.map((svc) => getServiceNames(svc, install.version)) : [];
 
     const writer = new CodeBlockWriter();
 
+    // imports
     genImport(writer, "requireService", opts.corePackage.name, opts.language);
 
     if (opts.language === "typescript") {

src/generate/index.ts

@@ -2,15 +2,15 @@ import { CommandModule } from "yargs";
 import * as fs from "fs";
 import { logger } from "../utils/log";
 import { directoryExists } from "../utils/fs";
-import { Installation, ProductionInstallation, readInstallInfo } from "../utils/installation";
+import { Installation, readInstallInfo } from "../utils/installation";
 import { corePackages } from "../nodecgIOVersions";
 import { GenerationOptions, promptGenerationOpts } from "./prompt";
 import { runNpmBuild, runNpmInstall } from "../utils/npm";
 import { genExtension } from "./extension";
 import { findNodeCGDirectory, getNodeCGIODirectory } from "../utils/nodecgInstallation";
 import { genDashboard, genGraphic } from "./panel";
 import { genTsConfig } from "./tsConfig";
-import { writeBundleFile, yellowGenerateCommand, yellowInstallCommand } from "./utils";
+import { writeBundleFile, yellowInstallCommand } from "./utils";
 import { genPackageJson } from "./packageJson";
 
 export const generateModule: CommandModule = {
@@ -40,19 +40,17 @@ export const generateModule: CommandModule = {
 };
 
 /**
- * Ensures that a installation can be used to generate bundles, meaning nodecg-io is actually installed,
- * is not a dev install and has some services installed that can be used.
+ * Ensures that a installation can be used to generate bundles, meaning nodecg-io is actually installed
+ * including at least one service that can be used for generating a bundle.
  * Throws an error if the installation cannot be used to generate a bundle with an explanation.
  */
-export function ensureValidInstallation(install: Installation | undefined): install is ProductionInstallation {
+export function ensureValidInstallation(install: Installation | undefined): install is Installation {
     if (install === undefined) {
         throw new Error(
             "nodecg-io is not installed to your local nodecg install.\n" +
                 `Please install it first using this command: ${yellowInstallCommand}`,
         );
-    } else if (install.dev) {
-        throw new Error(`You cannot use ${yellowGenerateCommand} together with a development installation.`);
-    } else if (install.packages.length <= corePackages.length) {
+    } else if (install.dev === false && install.packages.length <= corePackages.length) {
         // just has core packages without any services installed.
         throw new Error(
             `You first need to have at least one service installed to generate a bundle.\n` +
@@ -63,7 +61,7 @@ export function ensureValidInstallation(install: Installation | undefined): inst
     return true;
 }
 
-export async function generateBundle(opts: GenerationOptions, install: ProductionInstallation): Promise<void> {
+export async function generateBundle(opts: GenerationOptions, install: Installation): Promise<void> {
     // Create dir if necessary
     if (!(await directoryExists(opts.bundlePath))) {
         await fs.promises.mkdir(opts.bundlePath);
@@ -80,7 +78,7 @@ export async function generateBundle(opts: GenerationOptions, install: Productio
     }
 
     // All of these calls only generate files if they are set accordingly in the GenerationOptions
-    await genPackageJson(opts);
+    await genPackageJson(opts, install);
     await genTsConfig(opts);
     await genGitIgnore(opts);
     await genExtension(opts, install);

src/generate/packageJson.ts

@@ -4,6 +4,10 @@ import { getLatestPackageVersion } from "../utils/npm";
 import { genNodeCGDashboardConfig, genNodeCGGraphicConfig } from "./panel";
 import { SemVer } from "semver";
 import { writeBundleFile } from "./utils";
+import { Installation } from "../utils/installation";
+
+// Loaction where the development tarballs are hosted.
+export const developmentPublishRootUrl = "https://codeoverflow-org.github.io/nodecg-io-publish/";
 
 /**
  * A dependency on a npm package. First field is the package name and the second field is the version.
@@ -13,25 +17,25 @@ type Dependency = [string, string];
 /**
  * Generates the whole package.json file for the bundle.
  *
- * @param nodecgDir the directory in which nodecg is installed
  * @param opts the options that the user chose for the bundle.
+ * @param install the nodecg-io installation that will used to get the versions of the various packages.
  */
-export async function genPackageJson(opts: GenerationOptions): Promise<void> {
-    const serviceDeps: Dependency[] = opts.servicePackages.map((pkg) => [pkg.name, addSemverCaret(pkg.version)]);
+export async function genPackageJson(opts: GenerationOptions, install: Installation): Promise<void> {
+    const serviceDeps = opts.servicePackages.map((pkg) => getNodecgIODependency(pkg.name, pkg.version, install));
 
     const content = {
         name: opts.bundleName,
         version: opts.version.version,
         private: true,
         nodecg: {
-            compatibleRange: addSemverCaret("1.4.0"),
-            bundleDependencies: Object.fromEntries(serviceDeps),
+            compatibleRange: "^1.4.0",
+            bundleDependencies: Object.fromEntries(opts.servicePackages.map((pkg) => [pkg.name, `^${pkg.version}`])),
             graphics: genNodeCGGraphicConfig(opts),
             dashboardPanels: genNodeCGDashboardConfig(opts),
         },
         // These scripts are for compiling TS and thus are only needed when generating a TS bundle
         scripts: genScripts(opts),
-        dependencies: Object.fromEntries(await genDependencies(opts, serviceDeps)),
+        dependencies: Object.fromEntries(await genDependencies(opts, serviceDeps, install)),
     };
 
     await writeBundleFile(content, opts.bundlePath, "package.json");
@@ -45,8 +49,8 @@ export async function genPackageJson(opts: GenerationOptions): Promise<void> {
  * @param nodecgDir the directory in which nodecg is installed
  * @return the dependencies for a bundle with the given options.
  */
-async function genDependencies(opts: GenerationOptions, serviceDeps: Dependency[]) {
-    const core = [opts.corePackage.name, addSemverCaret(opts.corePackage.version)];
+async function genDependencies(opts: GenerationOptions, serviceDeps: Dependency[], install: Installation) {
+    const core = getNodecgIODependency(opts.corePackage.name, opts.corePackage.version, install);
 
     if (opts.language === "typescript") {
         // For typescript we need core, all services (for typings) and special packages like ts itself or node typings.
@@ -73,9 +77,9 @@ async function genTypeScriptDependencies(opts: GenerationOptions): Promise<Depen
     ]);
 
     return [
-        [opts.nodeeCGTypingsPackage, addSemverCaret(nodecgVersion)],
-        ["@types/node", addSemverCaret(latestNodeTypes)],
-        ["typescript", addSemverCaret(latestTypeScript)],
+        [opts.nodeeCGTypingsPackage, `^${nodecgVersion}`],
+        ["@types/node", `^${latestNodeTypes}`],
+        ["typescript", `^${latestTypeScript}`],
     ];
 }
 
@@ -86,6 +90,7 @@ async function genTypeScriptDependencies(opts: GenerationOptions): Promise<Depen
  */
 function genScripts(opts: GenerationOptions) {
     if (opts.language !== "typescript") {
+        // For JS we don't need any scripts to build anythiing.
         return undefined;
     }
 
@@ -98,11 +103,14 @@ function genScripts(opts: GenerationOptions) {
 }
 
 /**
- * Adds the semver caret operator to a given version to allow or minor and patch updates by npm.
- *
- * @param version the base version
- * @return the version with the semver caret operator in front.
+ * Builds the npm dependency for the package with the passed name and version.
+ * If this is a production install it will be from the npm registry and
+ * if it is a development install it will be from a tarball of the nodecg-io-publish repository.
  */
-function addSemverCaret(version: string | SemVer): string {
-    return `^${version}`;
+function getNodecgIODependency(packageName: string, version: string | SemVer, install: Installation): Dependency {
+    if (install.dev) {
+        return [packageName, `${developmentPublishRootUrl}${packageName}-${version}.tgz`];
+    } else {
+        return [packageName, `^${version}`];
+    }
 }

src/generate/prompt.ts

@@ -2,11 +2,12 @@ import * as semver from "semver";
 import * as inquirer from "inquirer";
 import * as path from "path";
 import { directoryExists } from "../utils/fs";
-import { ProductionInstallation } from "../utils/installation";
+import { Installation } from "../utils/installation";
 import { getServicesFromInstall } from "../install/prompt";
 import { yellowInstallCommand } from "./utils";
-import { NpmPackage } from "../utils/npm";
+import { findNpmPackages, NpmPackage } from "../utils/npm";
 import { corePackage } from "../nodecgIOVersions";
+import { getNodeCGIODirectory } from "../utils/nodecgInstallation";
 
 /**
  * Describes all options for bundle generation a user has answered with inside the inquirer prompt
@@ -36,16 +37,15 @@ export type CodeLanguage = "typescript" | "javascript";
 
 const kebabCaseRegex = /^([a-z][a-z0-9]*)(-[a-z0-9]+)*$/;
 
-export async function promptGenerationOpts(
-    nodecgDir: string,
-    install: ProductionInstallation,
-): Promise<GenerationOptions> {
+export async function promptGenerationOpts(nodecgDir: string, install: Installation): Promise<GenerationOptions> {
     const defaultBundleDir = path.join(nodecgDir, "bundles");
     // if we are already in a bundle directory we use the name of the directory as a bundle name and the corresponding bundle dir
     const inBundleDir = path.dirname(process.cwd()) === defaultBundleDir;
     const bundleName = inBundleDir ? path.basename(process.cwd()) : undefined;
     const bundleDir = inBundleDir ? path.dirname(process.cwd()) : defaultBundleDir;
 
+    const installedPackages = install.dev ? await findNpmPackages(getNodeCGIODirectory(nodecgDir)) : install.packages;
+
     const opts: PromptedGenerationOptions = await inquirer.prompt([
         {
             type: "input",
@@ -74,13 +74,20 @@ export async function promptGenerationOpts(
             validate: validateVersion,
             filter: (ver) => new semver.SemVer(ver),
         },
-        {
-            type: "checkbox",
-            name: "services",
-            message: `Which services would you like to use? (they must be installed through ${yellowInstallCommand} first)`,
-            choices: getServicesFromInstall(install, install.version),
-            validate: validateServiceSelection,
-        },
+        !install.dev
+            ? {
+                  type: "checkbox",
+                  name: "services",
+                  message: `Which services would you like to use? (they must be installed through ${yellowInstallCommand} first)`,
+                  choices: getServicesFromInstall(installedPackages, install.version),
+                  validate: validateServiceSelection,
+              }
+            : {
+                  type: "input",
+                  name: "services",
+                  message: `Which services would you like to use? (comma separated)`,
+                  filter: (servicesString) => servicesString.split(","),
+              },
         {
             type: "list",
             name: "language",
@@ -99,7 +106,7 @@ export async function promptGenerationOpts(
         },
     ]);
 
-    return computeGenOptsFields(opts, install);
+    return computeGenOptsFields(opts, install, installedPackages);
 }
 
 // region prompt validation
@@ -151,9 +158,10 @@ function validateServiceSelection(services: string[]): true | string {
  */
 export function computeGenOptsFields(
     opts: PromptedGenerationOptions,
-    install: ProductionInstallation,
+    install: Installation,
+    installedPackages: NpmPackage[],
 ): GenerationOptions {
-    const corePkg = install.packages.find((pkg) => pkg.name === corePackage);
+    const corePkg = installedPackages.find((pkg) => pkg.name === corePackage);
     if (corePkg === undefined) {
         throw new Error("Core package in installation info could not be found.");
     }
@@ -162,7 +170,7 @@ export function computeGenOptsFields(
         ...opts,
         corePackage: corePkg,
         servicePackages: opts.services.map((svc) => {
-            const svcPackage = install.packages.find((pkg) => pkg.name.endsWith(svc));
+            const svcPackage = installedPackages.find((pkg) => pkg.name.endsWith(svc));
 
             if (svcPackage === undefined) {
                 throw new Error(`Service ${svc} has no corresponding package in the passed installation.`);

src/generate/utils.ts

@@ -6,7 +6,6 @@ import * as chalk from "chalk";
 
 // Colored commands for logging purposes.
 export const yellowInstallCommand = chalk.yellow("nodecg-io install");
-export const yellowGenerateCommand = chalk.yellow("nodecg-io generate");
 
 /**
  * Writes a file for a bundle.

src/install/prompt.ts

@@ -1,4 +1,4 @@
-import { Installation, ProductionInstallation } from "../utils/installation";
+import { Installation } from "../utils/installation";
 import * as inquirer from "inquirer";
 import { getHighestPatchVersion, getMinorVersions, NpmPackage } from "../utils/npm";
 import * as semver from "semver";
@@ -85,7 +85,7 @@ export async function promptForInstallInfo(
             when: (x: PromptInput) => x.version !== developmentVersion,
             default: (x: PromptInput) => {
                 if (!currentProd) return;
-                return getServicesFromInstall(currentProd, x.version);
+                return getServicesFromInstall(currentProd.packages, x.version);
             },
         },
     ]);
@@ -185,15 +185,15 @@ function getPackageSymlinks(version: string, pkgName: string) {
 }
 
 /**
- * Returns the list of installed services of a production installation.
- * @param install the installation info for which you want the list of installed services.
+ * Returns the list of installed services of a nodecg-io installation.
+ * @param installedPackages a array with all packages that are installed
  * @param targetVersion the version of nodecg-io that is installed
  * @returns the list of installed services (package names without the nodecg-io- prefix)
  */
-export function getServicesFromInstall(install: ProductionInstallation, targetVersion: string): string[] {
+export function getServicesFromInstall(installedPackages: NpmPackage[], targetVersion: string): string[] {
     const availableServices = getServicesForVersion(targetVersion);
 
-    const svcPackages = install.packages
+    const svcPackages = installedPackages
         // Exclude core packages, they are not a optional service, they are always required
         .filter((pkg) => !corePackages.find((corePkg) => pkg.name === corePkg))
         .map((pkg) => pkg.name.replace("nodecg-io-", ""))

src/utils/npm.ts

@@ -236,6 +236,48 @@ export function getSubPackages(allPackages: NpmPackage[], rootPkg: NpmPackage):
     return allPackages.filter((pkg) => pkg !== rootPkg && pkg.path.startsWith(rootPkg.path));
 }
 
+/**
+ * Recursively finds npm packages using {@link findNpmPackages} in the given directory.
+ */
+export async function findNpmPackages(basePath: string): Promise<NpmPackage[]> {
+    // If there is a package in this directory, get it
+    const pkg = await getNpmPackageFromPath(basePath);
+
+    // Enumerate sub directories and get any packages in these too
+    const subDirs = await fs.promises.readdir(basePath, { withFileTypes: true });
+    const subPackages = await Promise.all(
+        subDirs
+            .filter((f) => f.isDirectory())
+            .map((f) => f.name)
+            .filter((dir) => dir !== "node_modules") // dependencies, not interesting to us. Also waaaay to big to check, lol
+            .map((subDir) => findNpmPackages(path.join(basePath, subDir))),
+    );
+
+    return [pkg, ...subPackages.flat()].filter((p): p is NpmPackage => p !== undefined);
+}
+
+/**
+ * Gets the npm package that is located in the directory of the passed path.
+ * @param basePath the root directory of the package where the package.json resides in
+ * @returns if a package.json was found and the package is public, the npm package. Otherwise undefined
+ */
+async function getNpmPackageFromPath(basePath: string): Promise<NpmPackage | undefined> {
+    const packageJsonPath = `${basePath}/package.json`;
+    try {
+        const packageJson = await fs.promises.readFile(packageJsonPath, "utf8");
+        const pkg = JSON.parse(packageJson);
+        if (pkg.private) return undefined;
+
+        return {
+            name: pkg.name,
+            version: pkg.version,
+            path: basePath,
+        };
+    } catch (e) {
+        return undefined;
+    }
+}
+
 /**
  * Gets version of the installed npm by running "npm --version".
  * @returns the npm version or undefined if npm is not installed/not in $PATH.