diff --git a/README.md b/README.md index 74bec85..426bcda 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ This `README.md` is generated with `markdown-magic` [view the raw file](https:// - + ## Install **Via npm** @@ -127,7 +127,7 @@ In NPM scripts, `npm run docs` would run the markdown magic and parse all the `. }, ``` -If you have a `markdown.config.js` file where `markdown-magic` is invoked, it will automatically use that as the configuration unless otherwise specified by `--config` flag. +If you have an `md.config.js` or `markdown.config.js` file where `markdown-magic` is invoked, it will automatically use that as the configuration unless otherwise specified by `--config` flag. ### Running programmatically @@ -191,7 +191,7 @@ jobs: - [markdown-autodocs](https://github.com/dineshsonachalam/markdown-autodocs) - Auto-generate docs from code - + ## Syntax Examples There are various syntax options. Choose your favorite. @@ -669,7 +669,7 @@ const config = { return '**⊂◉‿◉つ**' }, lolz() { - return `This section was generated by the cli config markdown.config.js file` + return `This section was generated by the cli config md.config.js file` }, /* Match */ pluginExample: require('./plugin-example')({ addNewLine: true }), diff --git a/docs/1-getting-started.md b/docs/1-getting-started.md index df25606..5e10e20 100644 --- a/docs/1-getting-started.md +++ b/docs/1-getting-started.md @@ -18,9 +18,9 @@ npm install markdown-magic --save-dev Add comment blocks to your markdown files to define where content should be generated: ```md - + This content will be dynamically replaced from the remote url - + ``` Then run markdown-magic to process your files. @@ -44,7 +44,7 @@ md-magic Process specific files with custom config: ```bash -md-magic --path '**/*.md' --config ./config.file.js +md-magic --files '**/*.md' --config ./config.file.js ``` ### NPM Scripts Integration @@ -54,7 +54,7 @@ Add to your `package.json`: ```json { "scripts": { - "docs": "md-magic --path '**/*.md'" + "docs": "md-magic --files '**/*.md'" } } ``` @@ -63,7 +63,7 @@ Run with: `npm run docs` ### Configuration File -Create a `markdown.config.js` file in your project root for automatic configuration: +Create an `md.config.js` or `markdown.config.js` file in your project root for automatic configuration: ```js module.exports = { diff --git a/docs/2-syntax-reference.md b/docs/2-syntax-reference.md index 92afaa8..f67240d 100644 --- a/docs/2-syntax-reference.md +++ b/docs/2-syntax-reference.md @@ -14,11 +14,11 @@ All transform blocks follow this basic structure: ```md Content to be replaced - + ``` Where: -- `matchWord` is the opening keyword (default: `doc-gen`) +- `matchWord` is the opening keyword (default: `docs`) - `transformName` is the name of the transform to apply - `options` are optional parameters for the transform - Content between the tags will be replaced by the transform output @@ -30,9 +30,9 @@ Where: The simplest form - just specify the transform name and options: ```md - + content to be replaced - + ``` ### Curly Braces @@ -40,9 +40,9 @@ content to be replaced Wrap the transform name in curly braces: ```md - + content to be replaced - + ``` ### Square Brackets @@ -50,9 +50,9 @@ content to be replaced Wrap the transform name in square brackets: ```md - + content to be replaced - + ``` ### Parentheses @@ -60,9 +60,9 @@ content to be replaced Wrap the transform name in parentheses: ```md - + content to be replaced - + ``` ### Function Style @@ -70,34 +70,34 @@ content to be replaced Use function-like syntax with parentheses for options: ```md - content to be replaced - + ``` ## Option Formats ### String Options ```md - + ``` ### Boolean Options ```md - + ``` ### Array Options ```md - + ``` ### Multiple Options ```md - content to be replaced - + ``` ## Best Practices diff --git a/docs/3-plugin-development.md b/docs/3-plugin-development.md index c778f9b..ac41c97 100644 --- a/docs/3-plugin-development.md +++ b/docs/3-plugin-development.md @@ -9,21 +9,23 @@ Learn how to create custom transforms (plugins) for markdown-magic to extend its ## Transform Basics -A transform is a function that takes content and options, then returns processed content: +A transform is a function that takes a single API object and returns processed content: ```js // Basic transform structure -function myTransform(content, options, config) { +function myTransform(api) { + const { content, options, settings } = api // Process the content return processedContent } ``` -### Parameters +### API Parameters - `content`: The content between the comment blocks - `options`: Parsed options from the comment block -- `config`: Global markdown-magic configuration +- `srcPath`: Current source markdown file path +- `settings`: Global markdown-magic configuration ## Creating Your First Transform @@ -31,7 +33,7 @@ function myTransform(content, options, config) { ```js // transforms/greeting.js -module.exports = function greeting(content, options) { +module.exports = function greeting({ options }) { const name = options.name || 'World' return `Hello, ${name}!` } @@ -39,8 +41,8 @@ module.exports = function greeting(content, options) { Usage: ```md - - + + ``` Result: `Hello, Alice!` @@ -52,7 +54,7 @@ Result: `Hello, Alice!` const fs = require('fs') const path = require('path') -module.exports = function include(content, options) { +module.exports = function include({ content, options }) { if (!options.src) { throw new Error('include transform requires "src" option') } @@ -70,8 +72,8 @@ module.exports = function include(content, options) { Usage: ```md - - + + ``` ## Async Transforms @@ -82,7 +84,7 @@ For operations that require network requests or file system operations: // transforms/fetchContent.js const fetch = require('node-fetch') -module.exports = async function fetchContent(content, options) { +module.exports = async function fetchContent({ content, options }) { if (!options.url) { throw new Error('fetchContent requires "url" option') } @@ -108,7 +110,7 @@ module.exports = async function fetchContent(content, options) { // transforms/template.js const mustache = require('mustache') -module.exports = function template(content, options) { +module.exports = function template({ content, options }) { const template = options.template || content const data = { ...options.data, @@ -126,7 +128,7 @@ module.exports = function template(content, options) { // transforms/codeStats.js const fs = require('fs') -module.exports = function codeStats(content, options) { +module.exports = function codeStats({ options }) { if (!options.src) { throw new Error('codeStats requires "src" option') } @@ -149,7 +151,7 @@ module.exports = function codeStats(content, options) { ```js // transforms/tableOfContents.js -module.exports = function tableOfContents(content, options) { +module.exports = function tableOfContents({ content, options }) { const format = options.format || 'markdown' const maxDepth = parseInt(options.maxDepth) || 3 const minDepth = parseInt(options.minDepth) || 1 @@ -194,7 +196,7 @@ function extractHeadings(content, minDepth, maxDepth) { ### In Configuration File ```js -// markdown.config.js +// md.config.js const greeting = require('./transforms/greeting') const include = require('./transforms/include') @@ -215,7 +217,7 @@ import markdownMagic from 'markdown-magic' const config = { transforms: { - myTransform: (content, options) => { + myTransform: ({ content, options }) => { return `Processed: ${content}` } } @@ -231,7 +233,7 @@ markdownMagic('./docs/*.md', config) Markdown-magic automatically parses options from the comment block: ```md - - + + `.trim()) }) diff --git a/docs/4-advanced-usage.md b/docs/4-advanced-usage.md index f8618f5..36908c9 100644 --- a/docs/4-advanced-usage.md +++ b/docs/4-advanced-usage.md @@ -14,14 +14,14 @@ This guide covers advanced patterns and techniques for using markdown-magic effe You can use multiple transform blocks in sequence to build complex documentation: ```md - - + + - - + + - - + + ``` ### Conditional Processing @@ -32,7 +32,7 @@ Use custom transforms to conditionally include content: // In your config file module.exports = { transforms: { - conditional: (content, options) => { + conditional: ({ content, options }) => { if (process.env.NODE_ENV === options.env) { return options.content || content } @@ -43,8 +43,8 @@ module.exports = { ``` ```md - - + + ``` ## Error Handling @@ -57,7 +57,7 @@ Handle errors gracefully in your transforms: // Custom transform with error handling module.exports = { transforms: { - safeRemote: async (content, options) => { + safeRemote: async ({ content, options }) => { try { const response = await fetch(options.url) return await response.text() @@ -85,7 +85,7 @@ const validateOptions = (options, required) => { module.exports = { transforms: { - strictRemote: (content, options) => { + strictRemote: ({ content, options }) => { validateOptions(options, ['url']) // ... rest of transform logic } @@ -104,7 +104,7 @@ const cache = new Map() module.exports = { transforms: { - cachedRemote: async (content, options) => { + cachedRemote: async ({ content, options }) => { const cacheKey = `remote:${options.url}` if (cache.has(cacheKey)) { @@ -142,16 +142,16 @@ Promise.all(promises).then(results => { ### Environment-Specific Configs ```js -// markdown.config.js +// md.config.js const isDev = process.env.NODE_ENV === 'development' module.exports = { - matchWord: 'doc-gen', + matchWord: 'docs', outputDir: isDev ? './docs-dev' : './docs', transforms: { // Environment-specific transforms ...(isDev && { - debug: (content, options) => { + debug: ({ content, options }) => { console.log('Debug transform:', options) return content } @@ -170,7 +170,7 @@ import markdownMagic from 'markdown-magic' // Config for documentation const docsConfig = { - matchWord: 'doc-gen', + matchWord: 'docs', transforms: { /* docs-specific transforms */ } } @@ -192,7 +192,7 @@ markdownMagic('./README.md', readmeConfig) ```js module.exports = { transforms: { - packageInfo: (content, options) => { + packageInfo: ({ content, options }) => { const pkg = require('./package.json') return ` @@ -220,7 +220,7 @@ const mustache = require('mustache') module.exports = { transforms: { - template: (content, options) => { + template: ({ content, options }) => { const template = options.template || content const data = options.data || {} @@ -232,9 +232,9 @@ module.exports = { Usage: ```md - + Hello {{name}}, you are a {{role}}! - + ``` ## Testing Custom Transforms @@ -249,7 +249,7 @@ describe('myTransform', () => { it('should process content correctly', () => { const content = 'original' const options = { param: 'value' } - const result = myTransform(content, options) + const result = myTransform({ content, options }) expect(result).toBe('expected output') }) @@ -297,7 +297,7 @@ const debug = require('debug')('markdown-magic:custom') module.exports = { transforms: { - debugTransform: (content, options) => { + debugTransform: ({ content, options }) => { debug('Processing with options:', options) // ... transform logic debug('Result:', result) diff --git a/docs/contributing.md b/docs/contributing.md index f929e95..625d695 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -11,8 +11,8 @@ Thank you for your interest in contributing to markdown-magic! This guide will h ### Prerequisites -- Node.js (version 14 or higher) -- npm or yarn +- Node.js (version 18 or higher) +- pnpm (version 10 or higher) - Git ### Getting Started @@ -26,9 +26,7 @@ Thank you for your interest in contributing to markdown-magic! This guide will h 3. **Install dependencies**: ```bash - npm install - # or - yarn install + pnpm install ``` 4. **Create a feature branch**: @@ -41,12 +39,14 @@ Thank you for your interest in contributing to markdown-magic! This guide will h ``` markdown-magic/ ├── packages/ # Monorepo packages -│ ├── core/ # Core markdown-magic package -│ ├── cli/ # CLI package -│ └── transforms/ # Built-in transforms +│ ├── core/ # Core markdown-magic package +│ ├── block-parser/ # Comment block parser package +│ ├── block-replacer/ # Block replacement engine +│ ├── block-transformer/ # Transform orchestration engine +│ └── plugin-*/ # Optional plugin packages ├── examples/ # Usage examples ├── docs/ # Documentation -├── test/ # Test files +├── packages/**/_tests/ # Test files └── scripts/ # Build and development scripts ``` @@ -56,45 +56,31 @@ markdown-magic/ ```bash # Run all tests -npm test +pnpm test -# Run tests in watch mode -npm run test:watch - -# Run tests for specific package -npm run test:core +# Run tests for all packages directly +pnpm -r test ``` -### Linting and Formatting - -```bash -# Run linter -npm run lint - -# Auto-fix linting issues -npm run lint:fix - -# Format code -npm run format -``` +### Types and Build ### Building ```bash # Build all packages -npm run build +pnpm build -# Build specific package -npm run build:core +# Generate declaration files where configured +pnpm types ``` ### Testing Your Changes 1. **Link locally** to test in other projects: ```bash - npm link + pnpm link --global cd /path/to/test-project - npm link markdown-magic + pnpm link --global markdown-magic ``` 2. **Run examples**: @@ -105,8 +91,8 @@ npm run build:core 3. **Test CLI**: ```bash - npm run cli -- --help - npm run cli -- --path './test/*.md' + pnpm --filter markdown-magic run cli -- --help + pnpm --filter markdown-magic run cli -- --files './docs/*.md' ``` ## Types of Contributions @@ -186,7 +172,7 @@ New built-in transforms should: - **ES6+ features** are encouraged - **2 spaces** for indentation -- **Semicolons** are required +- **Semicolons** are generally omitted - **Single quotes** for strings - **Trailing commas** in multiline structures @@ -221,12 +207,13 @@ export default function mainFunction() {} ```js /** * Transform function description - * @param {string} content - The content to transform - * @param {object} options - Transform options - * @param {object} config - Global configuration + * @param {object} api - Transform API payload + * @param {string} api.content - The content to transform + * @param {object} api.options - Transform options + * @param {object} api.settings - Global configuration * @returns {string|Promise} Transformed content */ -function myTransform(content, options, config) { +function myTransform({ content, options, settings }) { // Implementation details } ``` diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index b1fbfa4..63300fd 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -56,8 +56,8 @@ npm update markdown-magic 2. **Typo in transform name**: ```md - ✅ - ❌ (case sensitive) + ✅ + ❌ (case sensitive) ``` 3. **Transform file not found**: @@ -75,9 +75,9 @@ npm update markdown-magic 1. **Check comment syntax**: ```md - ✅ - ❌ (extra dash) - ✅ + ❌ (extra dash) + ✅ - ❌ (wrong match word) + ❌ (wrong match word) ``` 3. **Check file glob pattern**: ```bash # Make sure your files match the pattern - md-magic --path '**/*.md' # All .md files - md-magic --path './docs/*.md' # Only docs folder + md-magic --files '**/*.md' # All .md files + md-magic --files './docs/*.md' # Only docs folder ``` ### Options Not Parsed @@ -108,24 +108,24 @@ npm update markdown-magic ```md - + - ❌ (missing quotes) - ❌ (spaces around =) - ❌ (unmatched quotes) + ❌ (missing quotes) + ❌ (spaces around =) + ❌ (unmatched quotes) ``` **Complex options**: ```md - + - + - @@ -143,7 +143,7 @@ npm update markdown-magic ```bash # Test your glob pattern ls **/*.md # Check if files exist - md-magic --path '**/*.md' # Use quotes for glob + md-magic --files '**/*.md' # Use quotes for glob ``` 2. **Check working directory**: @@ -209,14 +209,14 @@ chmod 755 docs/ **Solutions**: 1. **Use correct filename**: - - `markdown.config.js` (auto-detected) - - `md.config.js` (auto-detected) + - `md.config.js` (auto-detected, preferred) + - `markdown.config.js` (auto-detected, legacy) - Or specify: `--config ./my-config.js` 2. **Check file location**: ```bash # Config should be in project root or specify path - ls -la markdown.config.js + ls -la md.config.js md-magic --config ./path/to/config.js ``` @@ -420,8 +420,8 @@ module.exports = function myTransform(content, options) { 3. **Optimize file patterns**: ```bash # More specific patterns are faster - md-magic --path './docs/*.md' # ✅ Specific directory - md-magic --path '**/*.md' # ❌ Searches everywhere + md-magic --files './docs/*.md' # ✅ Specific directory + md-magic --files '**/*.md' # ❌ Searches everywhere ``` ## Getting Help @@ -443,13 +443,13 @@ node --version npm --version # Debug output -DEBUG=markdown-magic:* md-magic --path './problematic-file.md' 2> debug.log +DEBUG=markdown-magic:* md-magic --files './problematic-file.md' 2> debug.log # File contents (if not sensitive) cat problematic-file.md # Configuration -cat markdown.config.js +cat md.config.js ``` ### Minimal Reproduction @@ -461,15 +461,15 @@ Create a minimal example that reproduces the issue: const markdownMagic = require('markdown-magic') const content = ` - - + + ` require('fs').writeFileSync('test.md', content) markdownMagic('test.md', { transforms: { - problemTransform: (content, options) => { + problemTransform: ({ content, options }) => { // Minimal version of your problem return 'result' } diff --git a/examples/generate-readme.js b/examples/generate-readme.js index f6f3cb1..6895796 100644 --- a/examples/generate-readme.js +++ b/examples/generate-readme.js @@ -95,7 +95,7 @@ const config = { return '**⊂◉‿◉つ**' }, lolz() { - return `This section was generated by the cli config markdown.config.js file` + return `This section was generated by the cli config md.config.js file` }, /* Match */ pluginExample: require('./plugin-example')({ addNewLine: true }), diff --git a/md.config.js b/md.config.js index d3eefa6..2dc3588 100644 --- a/md.config.js +++ b/md.config.js @@ -1,4 +1,4 @@ -/* CLI markdown.config.js file example */ +/* CLI md.config.js file example */ module.exports = { // handleOutputPath: (currentPath) => { // const newPath = 'x' + currentPath diff --git a/packages/block-parser/src/index.js b/packages/block-parser/src/index.js index 75cf88a..9a63173 100644 --- a/packages/block-parser/src/index.js +++ b/packages/block-parser/src/index.js @@ -173,6 +173,7 @@ const defaultOptions = { function parseBlocks(contents, opts = {}) { const _options = Object.assign({}, defaultOptions, opts) const { syntax, customPatterns, firstArgIsType } = _options + const getLineNumberAt = createLineNumberResolver(contents) // Extract regex source from open/close (handles RegExp objects and '/pattern/flags' strings) const openInfo = getRegexSource(opts.open !== undefined ? opts.open : _options.open) @@ -375,7 +376,7 @@ Details: const indent = spaces || '' const openStart = newMatches.index + indent.length const openEnd = openStart + fullComment.length - const lineNum = contents.substr(0, openStart).split('\n').length + const lineNum = getLineNumberAt(openStart) if (newMatches.index === newerRegex.lastIndex) { newerRegex.lastIndex++ @@ -439,8 +440,8 @@ Details: const openStart = newMatches.index + indent.length const openEnd = openStart + openTag.length const closeEnd = newerRegex.lastIndex - const lineOpen = contents.substr(0, openStart).split('\n').length - const lineClose = contents.substr(0, closeEnd).split('\n').length + const lineOpen = getLineNumberAt(openStart) + const lineClose = getLineNumberAt(closeEnd) const contentStart = openStart + openTag.length const contentEnd = contentStart + content.length @@ -533,8 +534,8 @@ Details: const closeEnd = newerRegex.lastIndex // const finIndentation = (lineOpen === lineClose) ? '' : indent - const lineOpen = contents.substr(0, openStart).split('\n').length - const lineClose = contents.substr(0, closeEnd).split('\n').length + const lineOpen = getLineNumberAt(openStart) + const lineClose = getLineNumberAt(closeEnd) const contentStart = openStart + openTag.length // + indent.length// - shift //+ indent.length const contentEnd = contentStart + content.length // + finIndentation.length // + shift @@ -634,6 +635,36 @@ function findLeadingIndent(str) { return (str.match(LEADING_INDENT_REGEX) || [])[1]?.length || 0 } +/** + * Build a fast line-number resolver for character offsets + * @param {string} input + * @returns {(position: number) => number} + */ +function createLineNumberResolver(input) { + const newlineIndexes = [] + for (let i = 0; i < input.length; i++) { + if (input.charCodeAt(i) === 10) { + newlineIndexes.push(i) + } + } + + return function getLineNumberAt(position) { + if (position <= 0) return 1 + + let low = 0 + let high = newlineIndexes.length + while (low < high) { + const mid = (low + high) >> 1 + if (newlineIndexes[mid] < position) { + low = mid + 1 + } else { + high = mid + } + } + return low + 1 + } +} + function verifyTagsBalanced(str, open, close) { const openCount = (str.match(open) || []).length // console.log('openCount', openCount) diff --git a/packages/block-replacer/src/index.js b/packages/block-replacer/src/index.js index 2d9e8c2..aab2053 100644 --- a/packages/block-replacer/src/index.js +++ b/packages/block-replacer/src/index.js @@ -16,6 +16,7 @@ const { blockTransformer } = require('comment-block-transformer') * @typedef {ProcessContentConfig & { * content?: string * srcPath?: string + * parsedBlocks?: import('comment-block-parser').ParseBlocksResult * outputPath?: string * dryRun?: boolean * patterns?: { @@ -82,11 +83,11 @@ async function processFile(opts = {}) { const outputDir = output.directory || opts.outputDir let srcPath = opts.srcPath - if (srcPath && content) { - throw new Error(`Can't set both "srcPath" & "content"`) - } let fileContents - if (content) { + if (typeof content === 'string' && srcPath) { + // Allow callers to provide preloaded content for srcPath files + fileContents = content + } else if (content) { const isFile = isValidFile(content) && content.indexOf('\n') === -1 srcPath = (isFile) ? content : undefined fileContents = (!isFile) ? content : undefined diff --git a/packages/block-replacer/test/index.test.js b/packages/block-replacer/test/index.test.js index 65322a6..928f9e6 100644 --- a/packages/block-replacer/test/index.test.js +++ b/packages/block-replacer/test/index.test.js @@ -192,20 +192,18 @@ test content assert.is(result.isChanged, false) }) -test('should handle both srcPath and content error', async () => { +test('should handle both srcPath and content using preloaded content', async () => { /** @type {ProcessFileOptions} */ const options = { - srcPath: '/some/path', - content: 'some content', + srcPath: '/some/nonexistent/path', + content: 'preloaded content', dryRun: true } - try { - await processFile(options) - assert.unreachable('Should have thrown an error') - } catch (error) { - assert.ok(error.message.includes('Can\'t set both "srcPath" & "content"')) - } + // When both srcPath and content are provided, content is used as preloaded + // file contents (caching optimization) — no error should be thrown. + const result = await processFile(options) + assert.ok(result, 'Should return a result without throwing') }) test('should handle file with output directory', async () => { diff --git a/packages/block-transformer/src/index.js b/packages/block-transformer/src/index.js index b254255..fdf72b1 100644 --- a/packages/block-transformer/src/index.js +++ b/packages/block-transformer/src/index.js @@ -71,6 +71,7 @@ const CLOSE_WORD = '/block' * @property {string} [srcPath] - The source path. * @property {string} [outputPath] - The output path. * @property {import('comment-block-parser').CustomPatterns} [customPatterns] - Custom regex patterns for open and close tags. + * @property {import('comment-block-parser').ParseBlocksResult} [parsedBlocks] - Optional pre-parsed block data to skip reparsing. */ /** @@ -112,18 +113,21 @@ async function blockTransformer(inputText, config) { // Don't default close - let undefined pass through to enable pattern mode in block-parser const close = opts.close !== undefined ? opts.close : (opts.open ? undefined : CLOSE_WORD) - let foundBlocks = {} - try { - foundBlocks = parseBlocks(inputText, { - syntax, - open, - close, - customPatterns, - firstArgIsType: true, - }) - } catch (e) { - const errMsg = (srcPath) ? `in ${srcPath}` : inputText - throw new Error(`${e.message}\nFix content in ${errMsg}\n`) + /** @type {import('comment-block-parser').ParseBlocksResult} */ + let foundBlocks = opts.parsedBlocks + if (!foundBlocks || !Array.isArray(foundBlocks.blocks)) { + try { + foundBlocks = parseBlocks(inputText, { + syntax, + open, + close, + customPatterns, + firstArgIsType: true, + }) + } catch (e) { + const errMsg = (srcPath) ? `in ${srcPath}` : inputText + throw new Error(`${e.message}\nFix content in ${errMsg}\n`) + } } diff --git a/packages/core/_tests/transform-code.test.js b/packages/core/_tests/transform-code.test.js index 201fb06..6a0e15c 100644 --- a/packages/core/_tests/transform-code.test.js +++ b/packages/core/_tests/transform-code.test.js @@ -6,6 +6,7 @@ const { test } = require('uvu') const assert = require('uvu/assert') const { markdownMagic } = require('../src') const { FIXTURE_DIR, MARKDOWN_FIXTURE_DIR, OUTPUT_DIR } = require('./config') +const TEMP_FIXTURE_DIR = path.join(FIXTURE_DIR, 'temp-code') function getNewFile(result) { if (!result.results || !result.results[0]) { @@ -16,6 +17,10 @@ function getNewFile(result) { const SILENT = true +function ensureDir(dir) { + fs.mkdirSync(dir, { recursive: true }) +} + test('CODE - local file inclusion', async () => { const fileName = 'transform-code.md' const filePath = path.join(MARKDOWN_FIXTURE_DIR, fileName) @@ -105,4 +110,30 @@ test('CODE - legacy colon syntax works', async () => { assert.ok(newContent.includes('module.exports.run'), 'legacy syntax processed correctly') }) +test('CODE - throws when id markers are missing', async () => { + const content = ` +original content +` + + ensureDir(TEMP_FIXTURE_DIR) + const tempFile = path.join(TEMP_FIXTURE_DIR, 'code-id-missing.md') + fs.writeFileSync(tempFile, content) + + let threw = false + try { + await markdownMagic(tempFile, { + open: 'docs', + close: '/docs', + outputDir: OUTPUT_DIR, + applyTransformsToSource: false, + silent: SILENT + }) + } catch (err) { + threw = true + assert.ok(err.message.includes('Missing MISSING_ID code section'), 'throws missing code section error') + } + + assert.ok(threw, 'should throw when CODE id markers are missing') +}) + test.run() diff --git a/packages/core/_tests/transform-remote.test.js b/packages/core/_tests/transform-remote.test.js index cc36f5d..eeaad55 100644 --- a/packages/core/_tests/transform-remote.test.js +++ b/packages/core/_tests/transform-remote.test.js @@ -89,6 +89,31 @@ original content assert.ok(threw, 'should throw on invalid URL') }) +test('remote - throws on HTTP 404 response', async () => { + const content = ` +original content +` + + ensureDir(TEMP_FIXTURE_DIR) + const tempFile = path.join(TEMP_FIXTURE_DIR, 'remote-404.md') + fs.writeFileSync(tempFile, content) + + let threw = false + try { + await markdownMagic(tempFile, { + open: 'docs', + close: '/docs', + outputDir: OUTPUT_DIR, + applyTransformsToSource: false, + silent: SILENT + }) + } catch (err) { + threw = true + assert.ok(err.message.includes('status 404') || err.message.includes('404'), 'throws HTTP status error') + } + assert.ok(threw, 'should throw on 404 response') +}) + test('remote - fetches raw markdown from GitHub', async () => { const content = ` original diff --git a/packages/core/package.json b/packages/core/package.json index a902f99..5383398 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -21,7 +21,7 @@ "types": "tsc --emitDeclarationOnly --outDir types", "docs": "node examples/generate-readme.js", "test": "uvu . '.test.([mc]js|[jt]sx?)$'", - "cli": "node ./cli.js --path 'README.md' --config ./markdown.config.js", + "cli": "node ./cli.js --files 'README.md' --config ./md.config.js", "bundle": "npm run bundle:all", "bundle:all": "npm run bundle:mac && npm run bundle:linux && npm run bundle:windows", "bundle:mac": "bun build ./cli.js --compile --minify --target=bun-darwin-arm64 --outfile dist/md-magic-darwin-arm64 && bun build ./cli.js --compile --minify --target=bun-darwin-x64 --outfile dist/md-magic-darwin-x64", diff --git a/packages/core/src/cli-run.js b/packages/core/src/cli-run.js index c233212..9402628 100644 --- a/packages/core/src/cli-run.js +++ b/packages/core/src/cli-run.js @@ -11,7 +11,7 @@ const { getGlobGroupsFromArgs } = require('./globparse') // const { uxParse } = require('./argparse/argparse') const argv = process.argv.slice(2) const cwd = process.cwd() -const defaultConfigPath = 'md.config.js' +const defaultConfigPaths = ['md.config.js', 'markdown.config.js'] /** * Render markdown with ANSI styling for terminal output @@ -100,6 +100,22 @@ async function getBaseDir(opts = {}) { return (gitDir) ? path.dirname(gitDir) : currentDir } +/** + * Find the first config file that exists in parent dirs + * @param {string} baseDir + * @param {Array} configPaths + * @returns {Promise} + */ +async function findFirstConfig(baseDir, configPaths = []) { + for (let i = 0; i < configPaths.length; i++) { + const configPath = configPaths[i] + const found = await findUp(baseDir, configPath) + if (found) { + return found + } + } +} + function findSingleDashStrings(arr) { return arr.filter(str => str.match(/^-[^-]/)) } @@ -110,8 +126,9 @@ async function runCli(options = {}, rawArgv) { Usage: md-magic [options] [files...] Options: - --files, --file Files or glob patterns to process - --config Path to config file (default: md.config.js) + --files, --file, --path + Files or glob patterns to process + --config Path to config file (default: md.config.js or markdown.config.js) --output Output directory --open Opening comment keyword (default: docs) --close Closing comment keyword (default: /docs) @@ -125,6 +142,7 @@ Options: Examples: md-magic README.md md-magic --files "**/*.md" + md-magic --path "**/*.md" # alias for --files md-magic --config ./my-config.js Stdin/stdout mode: @@ -210,14 +228,14 @@ Stdin/stdout mode: /** */ options.files = [] /* If raw args found, process them further */ - if (argv.length && (options._ && options._.length || (options.file || options.files))) { + if (argv.length && (options._ && options._.length || (options.file || options.files || options.path))) { // if (isGlob(argv[0])) { // console.log('glob', argv[0]) // options.glob = argv[0] // } const globParse = getGlobGroupsFromArgs(argv, { /* CLI args that should be glob keys */ - globKeys: ['files', 'file'] + globKeys: ['files', 'file', 'path'] }) const { globGroups, otherOpts } = globParse /* @@ -304,6 +322,10 @@ Stdin/stdout mode: if (globGroupByKey.files) { options.files = options.files.concat(globGroupByKey.files.values) } + if (globGroupByKey.path) { + options.files = options.files.concat(globGroupByKey.path.values) + delete options.path + } if (globGroupByKey['']) { options.files = options.files.concat(globGroupByKey[''].values) } @@ -327,6 +349,11 @@ Stdin/stdout mode: delete extraParse.files } + if (extraParse.path) { + options.files = options.files.concat(extraParse.path) + delete extraParse.path + } + if (extraParse['--files']) { options.files = options.files.concat(extraParse['--files']) delete extraParse['--files'] @@ -354,7 +381,7 @@ Stdin/stdout mode: configFile = opts.config } else { const baseDir = await getBaseDir() - configFile = await findUp(baseDir, defaultConfigPath) + configFile = await findFirstConfig(baseDir, defaultConfigPaths) } const config = (configFile) ? loadConfig(configFile) : {} const mergedConfig = { diff --git a/packages/core/src/index.dependency-graph.test.js b/packages/core/src/index.dependency-graph.test.js new file mode 100644 index 0000000..4043ea8 --- /dev/null +++ b/packages/core/src/index.dependency-graph.test.js @@ -0,0 +1,49 @@ +const { test } = require('uvu') +const assert = require('uvu/assert') +const { __private } = require('./') + +test('createDependencyGraph includes every known dependency edge', () => { + const blockItems = [ + { + id: '/docs/main.md', + blocks: [{}], + dependencies: ['/docs/dep-a.md', '/docs/dep-z.md'] + }, + { + id: '/docs/dep-a.md', + blocks: [{}], + dependencies: [] + }, + { + id: '/docs/dep-z.md', + blocks: [{}], + dependencies: [] + } + ] + + const { graph } = __private.createDependencyGraph(blockItems) + assert.ok(graph.find((edge) => edge[0] === '/docs/main.md' && edge[1] === '/docs/dep-a.md')) + assert.ok(graph.find((edge) => edge[0] === '/docs/main.md' && edge[1] === '/docs/dep-z.md')) + assert.is(graph.filter((edge) => edge[0] === '/docs/main.md').length, 2) +}) + +test('createDependencyGraph skips dependencies outside the file set', () => { + const blockItems = [ + { + id: '/docs/main.md', + blocks: [{}], + dependencies: ['/docs/dep-a.md', '/external/file.js'] + }, + { + id: '/docs/dep-a.md', + blocks: [{}], + dependencies: [] + } + ] + + const { graph } = __private.createDependencyGraph(blockItems) + assert.is(graph.length, 1) + assert.equal(graph[0], ['/docs/main.md', '/docs/dep-a.md']) +}) + +test.run() diff --git a/packages/core/src/index.js b/packages/core/src/index.js index d078a16..83a5487 100644 --- a/packages/core/src/index.js +++ b/packages/core/src/index.js @@ -344,22 +344,24 @@ async function markdownMagic(globOrOpts = {}, options = {}) { name: file, id: file, srcPath: file, - blocks: foundBlocks.blocks + blocks: foundBlocks.blocks, + parsedBlocks: foundBlocks } }) const blocks = blocksByPath.map((item) => { const dir = path.dirname(item.srcPath) - item.dependencies = [] + const dependencySet = new Set() item.blocks.forEach((block) => { if (block.options && block.options.src) { const resolvedPath = path.resolve(dir, block.options.src) // if (resolvedPath.match(/\.md$/)) { // console.log('resolvedPath', resolvedPath) - item.dependencies = item.dependencies.concat(resolvedPath) + dependencySet.add(resolvedPath) //} } }) + item.dependencies = Array.from(dependencySet) return item }) @@ -381,23 +383,23 @@ async function markdownMagic(globOrOpts = {}, options = {}) { /** */ // Convert items into a format suitable for toposort - const graph = blocks - .filter((item) => item.blocks && item.blocks.length) - .map((item) => { - return [ item.id, ...item.dependencies ] - }) + const { itemsWithBlocks, itemIds, graph } = createDependencyGraph(blocks) // console.log('graph', graph) // Perform the topological sort and reverse for execution order - const sortedIds = toposort(graph).reverse(); + const sortedIds = graph.length ? toposort.array(itemIds, graph).reverse() : itemIds // Reorder items based on sorted ids - const sortedItems = sortedIds.map(id => blocks.find(item => item.id === id)).filter(Boolean); + const itemById = new Map(itemsWithBlocks.map((item) => [item.id, item])) + const sortedItems = sortedIds.map((id) => itemById.get(id)).filter(Boolean) // topoSort(blocks) const orderedFiles = sortedItems.map((block) => block.id) // console.log('sortedItems', sortedItems) // console.log('orderedFiles', orderedFiles) + const fileContentByPath = new Map(files.map((file, i) => [file, fileContents[i]])) + const parsedBlocksByPath = new Map(blocksByPath.map((item) => [item.id, item.parsedBlocks])) + const processedFiles = [] await asyncForEach(orderedFiles, async (file) => { // logger('file', file) @@ -424,6 +426,8 @@ async function markdownMagic(globOrOpts = {}, options = {}) { // logger('newPath', newPath) const result = await processFile({ ...opts, + content: fileContentByPath.get(file), + parsedBlocks: parsedBlocksByPath.get(file), patterns, open, close, @@ -787,6 +791,29 @@ function changedFiles(files) { return files.filter(({ isChanged }) => isChanged) } +/** + * Create graph data for deterministic dependency ordering + * @param {Array<{id: string, blocks: Array, dependencies?: Array}>} blockItems + * @returns {{itemsWithBlocks: Array, itemIds: Array, graph: Array<[string, string]>}} + */ +function createDependencyGraph(blockItems = []) { + const itemsWithBlocks = blockItems.filter((item) => item.blocks && item.blocks.length) + const itemIds = itemsWithBlocks.map((item) => item.id) + const itemIdSet = new Set(itemIds) + const graph = itemsWithBlocks + .flatMap((item) => { + return (item.dependencies || []) + .filter((dependency) => itemIdSet.has(dependency)) + .map((dependency) => [item.id, dependency]) + }) + + return { + itemsWithBlocks, + itemIds, + graph + } +} + async function asyncForEach(array, callback) { for (let index = 0; index < array.length; index++) { await callback(array[index], index, array) @@ -802,5 +829,8 @@ module.exports = { parseMarkdown, blockTransformer, processFile, - stringUtils + stringUtils, + __private: { + createDependencyGraph + } } \ No newline at end of file diff --git a/packages/core/src/transforms/code/index.js b/packages/core/src/transforms/code/index.js index 022079b..df468c4 100644 --- a/packages/core/src/transforms/code/index.js +++ b/packages/core/src/transforms/code/index.js @@ -158,23 +158,29 @@ module.exports = async function CODE(api) { /* Check for Id */ if (id) { - const lines = code.split("\n") - const startLineIndex = lines.findIndex(line => line.includes(`CODE_SECTION:${id}:START`)); - const startLine = startLineIndex !== -1 ? startLineIndex : 0; + const lines = code.split('\n') + const startLineIndex = lines.findIndex((line) => line.includes(`CODE_SECTION:${id}:START`)) + const endLineIndex = lines.findIndex((line) => line.includes(`CODE_SECTION:${id}:END`)) + + if (startLineIndex === -1 || endLineIndex === -1) { + throw new Error(`Missing ${id} code section from ${codeFilePath}`) + } + + if (endLineIndex <= startLineIndex) { + throw new Error(`Invalid ${id} code section in ${codeFilePath}. End marker must be after start marker`) + } - const endLineIndex = lines.findIndex(line => line.includes(`CODE_SECTION:${id}:END`)); - const endLine = endLineIndex !== -1 ? endLineIndex : lines.length - 1; // console.log('startLine', startLine) // console.log('endLine', endLine) - if (startLine === -1 && endLine === -1) { - throw new Error(`Missing ${id} code section from ${codeFilePath}`) + const selectedLines = lines.slice(startLineIndex + 1, endLineIndex) + + if (!selectedLines.length) { + throw new Error(`Empty ${id} code section in ${codeFilePath}`) } - - const selectedLines = lines.slice(startLine + 1, endLine) - - const firstMatch = selectedLines[0] && selectedLines[0].match(/^(\s*)/); - const trimBy = firstMatch && firstMatch[1] ? firstMatch[1].length : 0; - const newValue = `${selectedLines.map(line => line.substring(trimBy).replace(/^\/\/ CODE_SECTION:INCLUDE /g, "")).join("\n")}` + + const firstMatch = selectedLines[0] && selectedLines[0].match(/^(\s*)/) + const trimBy = firstMatch && firstMatch[1] ? firstMatch[1].length : 0 + const newValue = `${selectedLines.map((line) => line.substring(trimBy).replace(/^\/\/ CODE_SECTION:INCLUDE /g, '')).join('\n')}` // console.log('newValue', newValue) code = newValue } diff --git a/packages/core/src/utils/remoteRequest.js b/packages/core/src/utils/remoteRequest.js index 67f6e10..549371e 100644 --- a/packages/core/src/utils/remoteRequest.js +++ b/packages/core/src/utils/remoteRequest.js @@ -1,28 +1,52 @@ const fetch = require('node-fetch') function formatUrl(url = '') { - return url.match(/^https?:\/\//) ? url : `https://${url}` + if (typeof url !== 'string') return '' + const trimmed = url.trim() + if (!trimmed) return '' + return trimmed.match(/^https?:\/\//) ? trimmed : `https://${trimmed}` } async function remoteRequest(url, settings = {}, srcPath) { - let body const finalUrl = formatUrl(url) + const fixText = srcPath ? `\nFix "${url}" value in ${srcPath}` : '' + if (!finalUrl) { + const msg = `Invalid URL "${url}"${fixText}` + if (settings.failOnMissingRemote) { + throw new Error(msg) + } + console.log(msg) + return + } + // ignore demo url todo remove one day if (finalUrl === 'http://url-to-raw-md-file.md') { return } + + let response try { - const res = await fetch(finalUrl) - body = await res.text() + response = await fetch(finalUrl) } catch (e) { console.log(`⚠️ WARNING: REMOTE URL "${finalUrl}" NOT FOUND`) - const msg = (e.message || '').split('\n')[0] + `\nFix "${url}" value in ${srcPath}` + const msg = (e.message || '').split('\n')[0] + fixText console.log(msg) if (settings.failOnMissingRemote) { throw new Error(msg) } + return + } + + if (!response.ok) { + const msg = `Remote request failed with status ${response.status} (${response.statusText}) for "${finalUrl}"${fixText}` + console.log(`⚠️ WARNING: ${msg}`) + if (settings.failOnMissingRemote) { + throw new Error(msg) + } + return } - return body + + return response.text() } module.exports = { diff --git a/update-syntax.js b/update-syntax.js index 1e8b64c..e90d863 100644 --- a/update-syntax.js +++ b/update-syntax.js @@ -5,10 +5,10 @@ * to the new 'docs' syntax. It serves as an example of how to use the * generic migration utilities. * - * For creating custom migrations, see packages/block-migrate package + * For creating custom migrations, see packages/block-migrator package */ -const { migrateDocGenToDocs } = require('./packages/block-migrate/src/index'); +const { migrateDocGenToDocs } = require('./packages/block-migrator/src/index') async function updateMarkdownFiles() { console.log('Migrating doc-gen syntax to docs...\n');