feat: add docs migrate command (#1220)

kanadgupta · erunion · web-flow · commit 67bd2ef684ef · 2025-04-21T17:40:43.000-05:00
fixes RM-12513 ## 🧰 Changes experimental alpha, not ready for public usage just yet. ## 🧬 QA & Testing i added a tiny bit of test coverage, will eventually want to look into how to test this using plugins. --------- Co-authored-by: Jon Ursenbach <jon@ursenba.ch> Co-authored-by: Jon Ursenbach <erunion@users.noreply.github.com>
diff --git a/__tests__/commands/docs/__snapshots__/migrate.test.ts.snap b/__tests__/commands/docs/__snapshots__/migrate.test.ts.snap
@@ -0,0 +1,21 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`rdme docs migrate > should error out if no path is passed 1`] = `
+{
+  "error": [Error: Missing 1 required arg:
+path  Path to a local Markdown file or folder of Markdown files.
+See more help with --help],
+  "stderr": "",
+  "stdout": "",
+}
+`;
+
+exports[`rdme docs migrate > should error out if no plugins are installed 1`] = `
+{
+  "error": [Error: This command requires a valid migration plugin.],
+  "stderr": "- 🔍 Looking for Markdown files in the \`__tests__/__fixtures__/docs/new-docs\` directory...
+✔ 🔍 Looking for Markdown files in the \`__tests__/__fixtures__/docs/new-docs\` directory... 1 file(s) found!
+",
+  "stdout": "",
+}
+`;
diff --git a/__tests__/commands/docs/migrate.test.ts b/__tests__/commands/docs/migrate.test.ts
@@ -0,0 +1,24 @@
+import { beforeAll, describe, expect, it } from 'vitest';
+
+import Command from '../../../src/commands/docs/migrate.js';
+import { runCommand, type OclifOutput } from '../../helpers/oclif.js';
+
+describe('rdme docs migrate', () => {
+  let run: (args?: string[]) => OclifOutput;
+
+  beforeAll(() => {
+    run = runCommand(Command);
+  });
+
+  it('should error out if no path is passed', async () => {
+    const output = await run();
+    expect(output).toMatchSnapshot();
+  });
+
+  it('should error out if no plugins are installed', async () => {
+    const output = await run(['__tests__/__fixtures__/docs/new-docs']);
+    expect(output).toMatchSnapshot();
+  });
+
+  it.todo('should load plugin and transform docs');
+});
diff --git a/src/commands/docs/migrate.ts b/src/commands/docs/migrate.ts
@@ -0,0 +1,189 @@
+import type { PluginHooks } from '../../lib/hooks/exported.js';
+
+import { Args, Flags, type Hook } from '@oclif/core';
+import chalk from 'chalk';
+import ora from 'ora';
+import { dir } from 'tmp-promise';
+
+import BaseCommand from '../../lib/baseCommand.js';
+import { fix, writeFixes } from '../../lib/frontmatter.js';
+import isCI from '../../lib/isCI.js';
+import { oraOptions } from '../../lib/logger.js';
+import promptTerminal from '../../lib/promptWrapper.js';
+import { fetchMappings, fetchSchema } from '../../lib/readmeAPIFetch.js';
+import { findPages } from '../../lib/readPage.js';
+
+const alphaNotice = 'This command is in an experimental alpha and is likely to change. Use at your own risk!';
+
+export default class DocsMigrateCommand extends BaseCommand<typeof DocsMigrateCommand> {
+  id = 'docs migrate' as const;
+
+  route = 'guides' as const;
+
+  static hidden = true;
+
+  static summary = `Migrates a directory of pages to the ReadMe Guides format.\n\n${alphaNotice}`;
+
+  static description =
+    "The path can either be a directory or a single Markdown file. The command will transform the Markdown using plugins and validate the frontmatter to conform to ReadMe's standards.";
+
+  static args = {
+    path: Args.string({ description: 'Path to a local Markdown file or folder of Markdown files.', required: true }),
+  };
+
+  static flags = {
+    out: Flags.string({
+      summary: 'The directory to write the migration output to. Defaults to a temporary directory.',
+    }),
+    'skip-validation': Flags.boolean({
+      description:
+        'Skips the validation of the Markdown files. Useful if this command is as part of a chain of commands that includes `docs upload`.',
+    }),
+  };
+
+  async run() {
+    const { path: rawPathInput }: { path: string } = this.args;
+    const { out: rawOutputDir, 'skip-validation': skipValidation } = this.flags;
+
+    const outputDir = rawOutputDir || (await dir({ prefix: 'rdme-migration-output' })).path;
+
+    let pathInput = rawPathInput;
+
+    // todo: fix this type once https://github.com/oclif/core/pull/1359 is merged
+    const fileScanHookResults: Hook.Result<PluginHooks['pre_markdown_file_scan']['return']> = await this.config.runHook(
+      'pre_markdown_file_scan',
+      { pathInput },
+    );
+
+    fileScanHookResults.successes.forEach(success => {
+      if (success.result) {
+        pathInput = success.result;
+      }
+    });
+
+    fileScanHookResults.failures.forEach(fail => {
+      if (fail.error && fail.error instanceof Error) {
+        throw new Error(`Error executing the \`${fail.plugin.name}\` plugin: ${fail.error.message}`);
+      }
+    });
+
+    let unsortedFiles = await findPages.call(this, pathInput);
+
+    let transformedByHooks = false;
+
+    // todo: fix this type once https://github.com/oclif/core/pull/1359 is merged
+    const validationHookResults: Hook.Result<PluginHooks['pre_markdown_validation']['return']> =
+      await this.config.runHook('pre_markdown_validation', { pages: unsortedFiles });
+
+    if (!validationHookResults.successes.length && !validationHookResults.failures.length) {
+      throw new Error('This command requires a valid migration plugin.');
+    }
+
+    validationHookResults.successes.forEach(success => {
+      if (success.result && success.result.length) {
+        transformedByHooks = true;
+        this.log(`🔌 ${success.result.length} Markdown files updated via the \`${success.plugin.name}\` plugin`);
+        unsortedFiles = success.result;
+      }
+    });
+
+    validationHookResults.failures.forEach(fail => {
+      if (fail.error && fail.error instanceof Error) {
+        throw new Error(`Error executing the \`${fail.plugin.name}\` plugin: ${fail.error.message}`);
+      }
+    });
+
+    // todo: either DRY this validation logic up or remove it entirely
+    if (!skipValidation) {
+      const validationSpinner = ora({ ...oraOptions() }).start('🔬 Validating frontmatter data...');
+
+      const schema = await fetchSchema.call(this);
+      const mappings = await fetchMappings.call(this);
+
+      // validate the files, prompt user to fix if necessary
+      const validationResults = unsortedFiles.map(file => {
+        this.debug(`validating frontmatter for ${file.filePath}`);
+        return fix.call(this, file.data, schema, mappings);
+      });
+
+      const filesWithIssues = validationResults.filter(result => result.hasIssues);
+      const filesWithFixableIssues = filesWithIssues.filter(result => result.changeCount);
+      const filesWithUnfixableIssues = filesWithIssues.filter(result => result.unfixableErrors.length);
+
+      if (filesWithIssues.length) {
+        validationSpinner.warn(`${validationSpinner.text} issues found in ${filesWithIssues.length} file(s).`);
+        if (filesWithFixableIssues.length) {
+          if (isCI()) {
+            throw new Error(
+              `${filesWithIssues.length} file(s) have issues. Please run \`${this.config.bin} ${this.id} ${pathInput} --dry-run\` in a non-CI environment to fix them.`,
+            );
+          }
+
+          const { confirm } = await promptTerminal([
+            {
+              type: 'confirm',
+              name: 'confirm',
+              message: `${filesWithFixableIssues.length} file(s) have issues that can be fixed automatically. Would you like to make these changes?`,
+            },
+          ]);
+
+          if (!confirm) {
+            throw new Error('Aborting fixes due to user input.');
+          }
+
+          const fileUpdateSpinner = ora({ ...oraOptions() }).start(
+            `📝 Writing file changes to the following directory: ${chalk.underline(outputDir)}...`,
+          );
+
+          const updatedFiles = unsortedFiles.map((file, index) => {
+            return writeFixes.call(this, file, validationResults[index].updatedData, outputDir);
+          });
+
+          fileUpdateSpinner.succeed(`${fileUpdateSpinner.text} ${updatedFiles.length} file(s) updated!`);
+
+          unsortedFiles = updatedFiles;
+        }
+
+        // also inform the user if there are files with issues that can't be fixed
+        if (filesWithUnfixableIssues.length) {
+          this.warn(
+            `${filesWithUnfixableIssues.length} file(s) have issues that cannot be fixed automatically. Please get in touch with us at support@readme.io if you need a hand.`,
+          );
+        }
+      } else if (transformedByHooks) {
+        validationSpinner.succeed(`${validationSpinner.text} no issues found!`);
+
+        const fileUpdateSpinner = ora({ ...oraOptions() }).start(
+          `📝 Writing the updated files to the following directory: ${chalk.underline(outputDir)}...`,
+        );
+
+        const updatedFiles = unsortedFiles.map((file, index) => {
+          return writeFixes.call(this, file, validationResults[index].updatedData, outputDir);
+        });
+
+        fileUpdateSpinner.succeed(`${fileUpdateSpinner.text} done!`);
+
+        unsortedFiles = updatedFiles;
+      } else {
+        validationSpinner.succeed(`${validationSpinner.text} no issues found!`);
+      }
+    } else {
+      this.debug('skipping validation');
+      if (transformedByHooks) {
+        const fileUpdateSpinner = ora({ ...oraOptions() }).start(
+          `📝 Writing the updated files to the following directory: ${chalk.underline(outputDir)}...`,
+        );
+
+        const updatedFiles = unsortedFiles.map(file => {
+          return writeFixes.call(this, file, file.data, outputDir);
+        });
+
+        fileUpdateSpinner.succeed(`${fileUpdateSpinner.text} done!`);
+
+        unsortedFiles = updatedFiles;
+      }
+    }
+
+    return { outputDir };
+  }
+}
diff --git a/src/index.ts b/src/index.ts
@@ -1,6 +1,7 @@
 import type { ValueOf } from 'type-fest';
 
 import ChangelogsCommand from './commands/changelogs.js';
+import DocsMigrateCommand from './commands/docs/migrate.js';
 import DocsUploadCommand from './commands/docs/upload.js';
 import LoginCommand from './commands/login.js';
 import LogoutCommand from './commands/logout.js';
@@ -27,6 +28,7 @@ export { default as prerun } from './lib/hooks/prerun.js';
 export const COMMANDS = {
   changelogs: ChangelogsCommand,
 
+  'docs:migrate': DocsMigrateCommand,
   'docs:upload': DocsUploadCommand,
 
   login: LoginCommand,
diff --git a/src/lib/readPage.ts b/src/lib/readPage.ts
@@ -1,11 +1,17 @@
 import type ChangelogsCommand from '../commands/changelogs.js';
+import type DocsMigrateCommand from '../commands/docs/migrate.js';
 import type DocsUploadCommand from '../commands/docs/upload.js';
 
 import crypto from 'node:crypto';
 import fs from 'node:fs';
+import fsPromises from 'node:fs/promises';
 import path from 'node:path';
 
 import grayMatter from 'gray-matter';
+import ora from 'ora';
+
+import { oraOptions } from './logger.js';
+import readdirRecursive from './readdirRecursive.js';
 
 export interface PageMetadata<T = Record<string, unknown>> {
   /**
@@ -35,8 +41,8 @@ export interface PageMetadata<T = Record<string, unknown>> {
 /**
  * Returns the content, matter and slug of the specified Markdown or HTML file
  */
-export default function readPage(
-  this: ChangelogsCommand | DocsUploadCommand,
+export function readPage(
+  this: ChangelogsCommand | DocsMigrateCommand | DocsUploadCommand,
   /**
    * path to the HTML/Markdown file
    * (file extension must end in `.html`, `.md`., or `.markdown`)
@@ -58,3 +64,58 @@ export default function readPage(
   const hash = crypto.createHash('sha1').update(rawFileContents).digest('hex');
   return { content, data, filePath, hash, slug };
 }
+
+/**
+ * Takes a path input and finds pages. If the path is a directory, it will recursively search for files with the specified extensions.
+ * If the path is a file, it will check if the file has a valid extension.
+ *
+ * Once the files are found, it reads each file and returns an array of page metadata objects (e.g., the parsed frontmatter data).
+ */
+export async function findPages(
+  this: ChangelogsCommand | DocsMigrateCommand | DocsUploadCommand,
+  pathInput: string,
+  allowedFileExtensions: string[] = ['.markdown', '.md', '.mdx'],
+) {
+  let files: string[];
+
+  const stat = await fsPromises.stat(pathInput).catch(err => {
+    if (err.code === 'ENOENT') {
+      throw new Error("Oops! We couldn't locate a file or directory at the path you provided.");
+    }
+    throw err;
+  });
+
+  if (stat.isDirectory()) {
+    const fileScanningSpinner = ora({ ...oraOptions() }).start(
+      `🔍 Looking for Markdown files in the \`${pathInput}\` directory...`,
+    );
+    // Filter out any files that don't match allowedFileExtensions
+    files = readdirRecursive(pathInput).filter(file =>
+      allowedFileExtensions.includes(path.extname(file).toLowerCase()),
+    );
+
+    if (!files.length) {
+      fileScanningSpinner.fail(`${fileScanningSpinner.text} no files found.`);
+      throw new Error(
+        `The directory you provided (${pathInput}) doesn't contain any of the following file extensions: ${allowedFileExtensions.join(
+          ', ',
+        )}.`,
+      );
+    }
+
+    fileScanningSpinner.succeed(`${fileScanningSpinner.text} ${files.length} file(s) found!`);
+  } else {
+    const fileExtension = path.extname(pathInput).toLowerCase();
+    if (!allowedFileExtensions.includes(fileExtension)) {
+      throw new Error(
+        `Invalid file extension (${fileExtension}). Must be one of the following: ${allowedFileExtensions.join(', ')}`,
+      );
+    }
+
+    files = [pathInput];
+  }
+
+  this.debug(`number of files: ${files.length}`);
+
+  return files.map(file => readPage.call(this, file));
+}
diff --git a/src/lib/readmeAPIFetch.ts b/src/lib/readmeAPIFetch.ts
@@ -1,6 +1,6 @@
 import type { SpecFileType } from './prepareOas.js';
 import type { CommandClass } from '../index.js';
-import type { CommandsThatSyncMarkdown } from './syncPagePath.js';
+import type { APIv2PageCommands } from './syncPagePath.js';
 import type { Hook } from '@oclif/core';
 import type { SchemaObject } from 'oas/types';
 
@@ -488,7 +488,7 @@ export async function fetchMappings(this: CommandClass['prototype']): Promise<Ma
 /**
  * Fetches the schema for the current route from the OpenAPI description for ReadMe API v2.
  */
-export async function fetchSchema(this: CommandsThatSyncMarkdown) {
+export async function fetchSchema(this: APIv2PageCommands) {
   const oas = await this.readmeAPIFetch('/openapi.json')
     .then(res => {
       if (!res.ok) {
diff --git a/src/lib/syncDocsPath.legacy.ts b/src/lib/syncDocsPath.legacy.ts
@@ -10,7 +10,7 @@ import toposort from 'toposort';
 import { APIv1Error } from './apiError.js';
 import readdirRecursive from './readdirRecursive.js';
 import { cleanAPIv1Headers, handleAPIv1Res, readmeAPIv1Fetch } from './readmeAPIFetch.js';
-import readPage from './readPage.js';
+import { readPage } from './readPage.js';
 
 /** API path within ReadMe to update (e.g. `docs`, `changelogs`, etc.) */
 type PageType = 'changelogs' | 'custompages' | 'docs';
diff --git a/src/lib/syncPagePath.ts b/src/lib/syncPagePath.ts
