diff --git a/src/context/yaml/index.ts b/src/context/yaml/index.ts index 565e8715..37c97b41 100644 --- a/src/context/yaml/index.ts +++ b/src/context/yaml/index.ts @@ -18,6 +18,65 @@ import { Assets, Config, Auth0APIClient, AssetTypes, KeywordMappings } from '../ import { filterOnlyIncludedResourceTypes } from '..'; import { preserveKeywords } from '../../keywordPreservation'; +// Custom YAML type for file includes +const includeType = new yaml.Type('!include', { + kind: 'scalar', + resolve: (data) => typeof data === 'string', + construct: (data) => { + // This will be handled during the actual loading process + return { __include: data }; + } +}); + +const schema = yaml.DEFAULT_SCHEMA.extend([includeType]); + +// Function to resolve includes with cycle detection +function resolveIncludes(content: string, basePath: string, mappings?: KeywordMappings, disableKeywordReplacement?: boolean, visitedFiles = new Set()): string { + const obj = yaml.load(content, { schema }); + return resolveIncludesInObject(obj, basePath, mappings, disableKeywordReplacement, visitedFiles); +} + +function resolveIncludesInObject(obj, basePath, mappings?: KeywordMappings, disableKeywordReplacement?: boolean, visitedFiles = new Set()) { + if (Array.isArray(obj)) { + return obj.map(item => resolveIncludesInObject(item, basePath, mappings, disableKeywordReplacement, visitedFiles)); + } + + if (obj && typeof obj === 'object') { + if (obj.__include) { + const filePath = path.resolve(basePath, obj.__include); + + if (visitedFiles.has(filePath)) { + throw new Error(`Circular include detected: ${filePath}`); + } + + if (fs.existsSync(filePath)) { + visitedFiles.add(filePath); + let content = fs.readFileSync(filePath, 'utf8'); + + // Apply keyword replacement to included file content if mappings are provided + if (mappings && !disableKeywordReplacement) { + content = keywordReplace(content, mappings); + } else if (mappings && disableKeywordReplacement) { + content = wrapArrayReplaceMarkersInQuotes(content, mappings); + } + + const result = resolveIncludesInObject(yaml.load(content, { schema }), path.dirname(filePath), mappings, disableKeywordReplacement, new Set(visitedFiles)); + visitedFiles.delete(filePath); + return result; + } + throw new Error(`Include file not found: ${filePath}`); + } + + const result = {}; + for (const [key, value] of Object.entries(obj)) { + result[key] = resolveIncludesInObject(value, basePath, mappings, disableKeywordReplacement, visitedFiles); + } + return result; + } + + return obj; +} + export default class YAMLContext { basePath: string; configFile: string; @@ -62,6 +121,10 @@ export default class YAMLContext { if (!isFile(toLoad)) { // try load not relative to yaml file toLoad = f; + if (!isFile(toLoad)) { + // try absolute path resolution + toLoad = path.resolve(f); + } } return loadFileAndReplaceKeywords(path.resolve(toLoad), { mappings: this.mappings, @@ -78,13 +141,13 @@ export default class YAMLContext { try { const fPath = path.resolve(this.configFile); log.debug(`Loading YAML from ${fPath}`); + const content = opts.disableKeywordReplacement + ? wrapArrayReplaceMarkersInQuotes(fs.readFileSync(fPath, 'utf8'), this.mappings) + : keywordReplace(fs.readFileSync(fPath, 'utf8'), this.mappings); + Object.assign( this.assets, - yaml.load( - opts.disableKeywordReplacement - ? wrapArrayReplaceMarkersInQuotes(fs.readFileSync(fPath, 'utf8'), this.mappings) - : keywordReplace(fs.readFileSync(fPath, 'utf8'), this.mappings) - ) || {} + resolveIncludes(content, path.dirname(fPath), this.mappings, opts.disableKeywordReplacement) ); } catch (err) { log.debug(err.stack); diff --git a/test/context/yaml/actions.test.js b/test/context/yaml/actions.test.js index bb7a47c3..eb97c1e9 100644 --- a/test/context/yaml/actions.test.js +++ b/test/context/yaml/actions.test.js @@ -66,6 +66,165 @@ describe('#YAML context actions', () => { expect(context.assets.actions).to.deep.equal(target); }); + it('should process YAML with includes', async () => { + const dir = path.join(testDataDir, 'yaml', 'includes'); + cleanThenMkdir(dir); + + const clientsYaml = ` +- name: "Test Client" + app_type: "spa" +- name: "Test M2M" + app_type: "non_interactive" + `; + const clientsFile = path.join(dir, 'clients.yaml'); + fs.writeFileSync(clientsFile, clientsYaml); + + const mainYaml = ` +tenant: + friendly_name: 'Test Tenant' + +clients: !include clients.yaml + `; + const mainFile = path.join(dir, 'tenant.yaml'); + fs.writeFileSync(mainFile, mainYaml); + + const config = { AUTH0_INPUT_FILE: mainFile }; + const context = new Context(config, mockMgmtClient()); + await context.loadAssetsFromLocal(); + + expect(context.assets.tenant).to.deep.equal({ + friendly_name: 'Test Tenant', + }); + expect(context.assets.clients).to.deep.equal([ + { + name: 'Test Client', + app_type: 'spa', + }, + { + name: 'Test M2M', + app_type: 'non_interactive', + }, + ]); + }); + + it('should handle nested includes', async () => { + const dir = path.join(testDataDir, 'yaml', 'nested-includes'); + cleanThenMkdir(dir); + + const rolesYaml = ` +- name: Admin + description: Administrator +- name: User + description: Regular User + `; + fs.writeFileSync(path.join(dir, 'roles.yaml'), rolesYaml); + + const mainYaml = ` +tenant: + friendly_name: 'Main Tenant' + +roles: !include roles.yaml + `; + fs.writeFileSync(path.join(dir, 'tenant.yaml'), mainYaml); + + const config = { AUTH0_INPUT_FILE: path.join(dir, 'tenant.yaml') }; + const context = new Context(config, mockMgmtClient()); + await context.loadAssetsFromLocal(); + + expect(context.assets.roles).to.deep.equal([ + { name: 'Admin', description: 'Administrator' }, + { name: 'User', description: 'Regular User' }, + ]); + }); + + it('should process logStreams with includes', async () => { + const dir = path.join(testDataDir, 'yaml', 'logstreams-includes'); + cleanThenMkdir(dir); + + const logStreamsYaml = ` +- name: LoggingSAAS + isPriority: false + filters: + - type: category + name: auth.login.fail + - type: category + name: auth.login.notification + - type: category + name: auth.login.success + - type: category + name: auth.logout.fail + sink: + httpContentFormat: JSONLINES + httpContentType: application/json + httpEndpoint: "##LOGGING_WEBHOOK_URL##" + type: http +- name: SIEM + isPriority: false + filters: + - type: category + name: auth.login.fail + - type: category + name: auth.login.notification + - type: category + name: auth.login.success + sink: + httpContentFormat: JSONLINES + httpContentType: application/json + httpEndpoint: "##SIEM_WEBHOOK_URL##" + type: http + `; + fs.writeFileSync(path.join(dir, 'logStreams.yaml'), logStreamsYaml); + + const mainYaml = ` +tenant: + friendly_name: 'Test Tenant' + +logStreams: !include logStreams.yaml + `; + fs.writeFileSync(path.join(dir, 'tenant.yaml'), mainYaml); + + const config = { + AUTH0_INPUT_FILE: path.join(dir, 'tenant.yaml'), + AUTH0_KEYWORD_REPLACE_MAPPINGS: { + LOGGING_WEBHOOK_URL: 'https://logging.com/inputs/test', + SIEM_WEBHOOK_URL: 'https://siem.example.com/webhook' + } + }; + const context = new Context(config, mockMgmtClient()); + await context.loadAssetsFromLocal(); + + expect(context.assets.logStreams).to.have.length(2); + expect(context.assets.logStreams[0]).to.deep.include({ + name: 'LoggingSAAS', + isPriority: false, + type: 'http' + }); + expect(context.assets.logStreams[0].sink.httpEndpoint).to.equal('https://logging.com/inputs/test'); + expect(context.assets.logStreams[1]).to.deep.include({ + name: 'SIEM', + isPriority: false, + type: 'http' + }); + expect(context.assets.logStreams[1].sink.httpEndpoint).to.equal('https://siem.example.com/webhook'); + }); + + it('should error on missing include file', async () => { + const dir = path.join(testDataDir, 'yaml', 'missing-include'); + cleanThenMkdir(dir); + + const mainYaml = ` +clients: !include missing.yaml + `; + fs.writeFileSync(path.join(dir, 'tenant.yaml'), mainYaml); + + const config = { AUTH0_INPUT_FILE: path.join(dir, 'tenant.yaml') }; + const context = new Context(config, mockMgmtClient()); + + await expect(context.loadAssetsFromLocal()).to.be.eventually.rejectedWith( + Error, + /Include file not found/ + ); + }); it('should dump actions', async () => { const dir = path.join(testDataDir, 'yaml', 'actionsDump'); cleanThenMkdir(dir); diff --git a/test/context/yaml/includes.test.js b/test/context/yaml/includes.test.js new file mode 100644 index 00000000..caa92c84 --- /dev/null +++ b/test/context/yaml/includes.test.js @@ -0,0 +1,120 @@ +import path from 'path'; +import fs from 'fs-extra'; +import { expect } from 'chai'; +import Context from '../../../src/context/yaml'; +import { cleanThenMkdir, testDataDir, mockMgmtClient } from '../../utils'; + +describe('#YAML includes cycle detection', () => { + it('should detect direct circular include', async () => { + const dir = path.resolve(testDataDir, 'yaml', 'circular-direct'); + cleanThenMkdir(dir); + + const mainFile = path.join(dir, 'main.yaml'); + fs.writeFileSync(mainFile, 'data: !include self.yaml'); + + const selfFile = path.join(dir, 'self.yaml'); + fs.writeFileSync(selfFile, 'value: !include self.yaml'); + + const config = { AUTH0_INPUT_FILE: mainFile }; + const context = new Context(config, mockMgmtClient()); + + await expect(context.loadAssetsFromLocal()).to.be.eventually.rejectedWith( + Error, + /Circular include detected/ + ); + }); + + it('should detect indirect circular include (A → B → A)', async () => { + const dir = path.resolve(testDataDir, 'yaml', 'circular-indirect'); + cleanThenMkdir(dir); + + const fileA = path.join(dir, 'a.yaml'); + fs.writeFileSync(fileA, 'data: !include b.yaml'); + + const fileB = path.join(dir, 'b.yaml'); + fs.writeFileSync(fileB, 'value: !include a.yaml'); + + const config = { AUTH0_INPUT_FILE: fileA }; + const context = new Context(config, mockMgmtClient()); + + await expect(context.loadAssetsFromLocal()).to.be.eventually.rejectedWith( + Error, + /Circular include detected/ + ); + }); + + it('should detect complex circular include (A → B → C → A)', async () => { + const dir = path.resolve(testDataDir, 'yaml', 'circular-complex'); + cleanThenMkdir(dir); + + const fileA = path.join(dir, 'a.yaml'); + fs.writeFileSync(fileA, 'data: !include b.yaml'); + + const fileB = path.join(dir, 'b.yaml'); + fs.writeFileSync(fileB, 'value: !include c.yaml'); + + const fileC = path.join(dir, 'c.yaml'); + fs.writeFileSync(fileC, 'final: !include a.yaml'); + + const config = { AUTH0_INPUT_FILE: fileA }; + const context = new Context(config, mockMgmtClient()); + + await expect(context.loadAssetsFromLocal()).to.be.eventually.rejectedWith( + Error, + /Circular include detected/ + ); + }); + + it('should allow valid includes without cycles', async () => { + const dir = path.resolve(testDataDir, 'yaml', 'valid-includes'); + cleanThenMkdir(dir); + + const mainFile = path.join(dir, 'main.yaml'); + fs.writeFileSync(mainFile, ` +tenant: + friendly_name: Test + data: !include shared.yaml +rules: !include rules.yaml +`); + + const sharedFile = path.join(dir, 'shared.yaml'); + fs.writeFileSync(sharedFile, 'shared_value: 42'); + + const rulesFile = path.join(dir, 'rules.yaml'); + fs.writeFileSync(rulesFile, '[]'); + + const config = { AUTH0_INPUT_FILE: mainFile }; + const context = new Context(config, mockMgmtClient()); + + await context.loadAssetsFromLocal(); + + expect(context.assets.tenant.friendly_name).to.equal('Test'); + expect(context.assets.tenant.data.shared_value).to.equal(42); + expect(context.assets.rules).to.deep.equal([]); + }); + + it('should allow same file included multiple times in different branches', async () => { + const dir = path.resolve(testDataDir, 'yaml', 'multiple-includes'); + cleanThenMkdir(dir); + + const mainFile = path.join(dir, 'main.yaml'); + fs.writeFileSync(mainFile, ` +tenant: + friendly_name: Test + config1: !include shared.yaml + config2: !include shared.yaml +`); + + const sharedFile = path.join(dir, 'shared.yaml'); + fs.writeFileSync(sharedFile, 'shared'); + + const config = { AUTH0_INPUT_FILE: mainFile }; + const context = new Context(config, mockMgmtClient()); + + await context.loadAssetsFromLocal(); + + expect(context.assets.tenant.friendly_name).to.equal('Test'); + expect(context.assets.tenant.config1).to.equal('shared'); + expect(context.assets.tenant.config2).to.equal('shared'); + }); +}); \ No newline at end of file