diff --git a/src/context/yaml/index.ts b/src/context/yaml/index.ts index 5a965cd9f..efe51a1af 100644 --- a/src/context/yaml/index.ts +++ b/src/context/yaml/index.ts @@ -18,6 +18,79 @@ import { Assets, Config, Auth0APIClient, AssetTypes, KeywordMappings } from '../ import { filterOnlyIncludedResourceTypes } from '..'; import { preserveKeywords } from '../../keywordPreservation'; +// Custom YAML type for file includes +const includeType = new yaml.Type('!include', { + kind: 'scalar', + resolve: (data) => typeof data === 'string', + construct: (data) => { + // This will be handled during the actual loading process + return { __include: data }; + }, +}); + +const schema = yaml.DEFAULT_SCHEMA.extend([includeType]); + +function prepareYamlContent( + content: string, + mappings: KeywordMappings, + disableKeywordReplacement: boolean +): string { + return disableKeywordReplacement + ? wrapArrayReplaceMarkersInQuotes(content, mappings) + : keywordReplace(content, mappings); +} + +function parseYaml(content: string) { + return yaml.load(content, { schema }); +} + +type LoadIncludedYaml = (filePath: string) => any; + +// Resolves !include directives only. Keyword handling is applied by the caller +// when preparing each file's raw content before parse. +function resolveIncludes( + obj: any, + basePath: string, + loadIncludedYaml: LoadIncludedYaml, + visitedFiles = new Set() +): any { + if (Array.isArray(obj)) { + return obj.map((item) => resolveIncludes(item, basePath, loadIncludedYaml, visitedFiles)); + } + + if (obj && typeof obj === 'object') { + if (obj.__include) { + const filePath = path.resolve(basePath, obj.__include); + + if (visitedFiles.has(filePath)) { + throw new Error(`Circular include detected: ${filePath}`); + } + + if (!fs.existsSync(filePath)) { + throw new Error(`Include file not found: ${filePath}`); + } + + visitedFiles.add(filePath); + const result = resolveIncludes( + loadIncludedYaml(filePath), + path.dirname(filePath), + loadIncludedYaml, + new Set(visitedFiles) + ); + visitedFiles.delete(filePath); + return result; + } + + const result = {}; + for (const [key, value] of Object.entries(obj)) { + result[key] = resolveIncludes(value, basePath, loadIncludedYaml, visitedFiles); + } + return result; + } + + return obj; +} + export default class YAMLContext { basePath: string; configFile: string; @@ -83,13 +156,18 @@ export default class YAMLContext { try { const fPath = path.resolve(this.configFile); log.debug(`Loading YAML from ${fPath}`); + + const prepareContent = (content: string) => + prepareYamlContent(content, this.mappings, opts.disableKeywordReplacement); + + const loadIncludedYaml: LoadIncludedYaml = (filePath) => + parseYaml(prepareContent(fs.readFileSync(filePath, 'utf8'))); + + const parsed = parseYaml(prepareContent(fs.readFileSync(fPath, 'utf8'))); + Object.assign( this.assets, - yaml.load( - opts.disableKeywordReplacement - ? wrapArrayReplaceMarkersInQuotes(fs.readFileSync(fPath, 'utf8'), this.mappings) - : keywordReplace(fs.readFileSync(fPath, 'utf8'), this.mappings) - ) || {} + resolveIncludes(parsed, path.dirname(fPath), loadIncludedYaml) || {} ); } catch (err) { log.debug(err.stack); diff --git a/test/context/yaml/actions.test.js b/test/context/yaml/actions.test.js index ac20bb601..d4acc9ba1 100644 --- a/test/context/yaml/actions.test.js +++ b/test/context/yaml/actions.test.js @@ -66,6 +66,169 @@ describe('#YAML context actions', () => { expect(context.assets.actions).to.deep.equal(target); }); + it('should process YAML with includes', async () => { + const dir = path.join(testDataDir, 'yaml', 'includes'); + cleanThenMkdir(dir); + + const clientsYaml = ` +- name: "Test Client" + app_type: "spa" +- name: "Test M2M" + app_type: "non_interactive" + `; + const clientsFile = path.join(dir, 'clients.yaml'); + fs.writeFileSync(clientsFile, clientsYaml); + + const mainYaml = ` +tenant: + friendly_name: 'Test Tenant' + +clients: !include clients.yaml + `; + const mainFile = path.join(dir, 'tenant.yaml'); + fs.writeFileSync(mainFile, mainYaml); + + const config = { AUTH0_INPUT_FILE: mainFile }; + const context = new Context(config, mockMgmtClient()); + await context.loadAssetsFromLocal(); + + expect(context.assets.tenant).to.deep.equal({ + friendly_name: 'Test Tenant', + }); + expect(context.assets.clients).to.deep.equal([ + { + name: 'Test Client', + app_type: 'spa', + }, + { + name: 'Test M2M', + app_type: 'non_interactive', + }, + ]); + }); + + it('should handle nested includes', async () => { + const dir = path.join(testDataDir, 'yaml', 'nested-includes'); + cleanThenMkdir(dir); + + const rolesYaml = ` +- name: Admin + description: Administrator +- name: User + description: Regular User + `; + fs.writeFileSync(path.join(dir, 'roles.yaml'), rolesYaml); + + const mainYaml = ` +tenant: + friendly_name: 'Main Tenant' + +roles: !include roles.yaml + `; + fs.writeFileSync(path.join(dir, 'tenant.yaml'), mainYaml); + + const config = { AUTH0_INPUT_FILE: path.join(dir, 'tenant.yaml') }; + const context = new Context(config, mockMgmtClient()); + await context.loadAssetsFromLocal(); + + expect(context.assets.roles).to.deep.equal([ + { name: 'Admin', description: 'Administrator' }, + { name: 'User', description: 'Regular User' }, + ]); + }); + + it('should process logStreams with includes', async () => { + const dir = path.join(testDataDir, 'yaml', 'logstreams-includes'); + cleanThenMkdir(dir); + + const logStreamsYaml = ` +- name: LoggingSAAS + isPriority: false + filters: + - type: category + name: auth.login.fail + - type: category + name: auth.login.notification + - type: category + name: auth.login.success + - type: category + name: auth.logout.fail + sink: + httpContentFormat: JSONLINES + httpContentType: application/json + httpEndpoint: "##LOGGING_WEBHOOK_URL##" + type: http +- name: SIEM + isPriority: false + filters: + - type: category + name: auth.login.fail + - type: category + name: auth.login.notification + - type: category + name: auth.login.success + sink: + httpContentFormat: JSONLINES + httpContentType: application/json + httpEndpoint: "##SIEM_WEBHOOK_URL##" + type: http + `; + fs.writeFileSync(path.join(dir, 'logStreams.yaml'), logStreamsYaml); + + const mainYaml = ` +tenant: + friendly_name: 'Test Tenant' + +logStreams: !include logStreams.yaml + `; + fs.writeFileSync(path.join(dir, 'tenant.yaml'), mainYaml); + + const config = { + AUTH0_INPUT_FILE: path.join(dir, 'tenant.yaml'), + AUTH0_KEYWORD_REPLACE_MAPPINGS: { + LOGGING_WEBHOOK_URL: 'https://logging.com/inputs/test', + SIEM_WEBHOOK_URL: 'https://siem.example.com/webhook', + }, + }; + const context = new Context(config, mockMgmtClient()); + await context.loadAssetsFromLocal(); + + expect(context.assets.logStreams).to.have.length(2); + expect(context.assets.logStreams[0]).to.deep.include({ + name: 'LoggingSAAS', + isPriority: false, + type: 'http', + }); + expect(context.assets.logStreams[0].sink.httpEndpoint).to.equal( + 'https://logging.com/inputs/test' + ); + expect(context.assets.logStreams[1]).to.deep.include({ + name: 'SIEM', + isPriority: false, + type: 'http', + }); + expect(context.assets.logStreams[1].sink.httpEndpoint).to.equal( + 'https://siem.example.com/webhook' + ); + }); + + it('should error on missing include file', async () => { + const dir = path.join(testDataDir, 'yaml', 'missing-include'); + cleanThenMkdir(dir); + + const mainYaml = ` +clients: !include missing.yaml + `; + fs.writeFileSync(path.join(dir, 'tenant.yaml'), mainYaml); + + const config = { AUTH0_INPUT_FILE: path.join(dir, 'tenant.yaml') }; + const context = new Context(config, mockMgmtClient()); + + await expect(context.loadAssetsFromLocal()).to.be.eventually.rejectedWith( + Error, + /Include file not found/ + ); + }); it('should dump actions', async () => { const dir = path.join(testDataDir, 'yaml', 'actionsDump'); cleanThenMkdir(dir); diff --git a/test/context/yaml/includes.test.js b/test/context/yaml/includes.test.js new file mode 100644 index 000000000..c88b38f4d --- /dev/null +++ b/test/context/yaml/includes.test.js @@ -0,0 +1,126 @@ +import path from 'path'; +import fs from 'fs-extra'; +import { expect } from 'chai'; +import Context from '../../../src/context/yaml'; +import { cleanThenMkdir, testDataDir, mockMgmtClient } from '../../utils'; + +describe('#YAML includes cycle detection', () => { + it('should detect direct circular include', async () => { + const dir = path.resolve(testDataDir, 'yaml', 'circular-direct'); + cleanThenMkdir(dir); + + const mainFile = path.join(dir, 'main.yaml'); + fs.writeFileSync(mainFile, 'data: !include self.yaml'); + + const selfFile = path.join(dir, 'self.yaml'); + fs.writeFileSync(selfFile, 'value: !include self.yaml'); + + const config = { AUTH0_INPUT_FILE: mainFile }; + const context = new Context(config, mockMgmtClient()); + + await expect(context.loadAssetsFromLocal()).to.be.eventually.rejectedWith( + Error, + /Circular include detected/ + ); + }); + + it('should detect indirect circular include (A → B → A)', async () => { + const dir = path.resolve(testDataDir, 'yaml', 'circular-indirect'); + cleanThenMkdir(dir); + + const fileA = path.join(dir, 'a.yaml'); + fs.writeFileSync(fileA, 'data: !include b.yaml'); + + const fileB = path.join(dir, 'b.yaml'); + fs.writeFileSync(fileB, 'value: !include a.yaml'); + + const config = { AUTH0_INPUT_FILE: fileA }; + const context = new Context(config, mockMgmtClient()); + + await expect(context.loadAssetsFromLocal()).to.be.eventually.rejectedWith( + Error, + /Circular include detected/ + ); + }); + + it('should detect complex circular include (A → B → C → A)', async () => { + const dir = path.resolve(testDataDir, 'yaml', 'circular-complex'); + cleanThenMkdir(dir); + + const fileA = path.join(dir, 'a.yaml'); + fs.writeFileSync(fileA, 'data: !include b.yaml'); + + const fileB = path.join(dir, 'b.yaml'); + fs.writeFileSync(fileB, 'value: !include c.yaml'); + + const fileC = path.join(dir, 'c.yaml'); + fs.writeFileSync(fileC, 'final: !include a.yaml'); + + const config = { AUTH0_INPUT_FILE: fileA }; + const context = new Context(config, mockMgmtClient()); + + await expect(context.loadAssetsFromLocal()).to.be.eventually.rejectedWith( + Error, + /Circular include detected/ + ); + }); + + it('should allow valid includes without cycles', async () => { + const dir = path.resolve(testDataDir, 'yaml', 'valid-includes'); + cleanThenMkdir(dir); + + const mainFile = path.join(dir, 'main.yaml'); + fs.writeFileSync( + mainFile, + ` +tenant: + friendly_name: Test + data: !include shared.yaml +rules: !include rules.yaml +` + ); + + const sharedFile = path.join(dir, 'shared.yaml'); + fs.writeFileSync(sharedFile, 'shared_value: 42'); + + const rulesFile = path.join(dir, 'rules.yaml'); + fs.writeFileSync(rulesFile, '[]'); + + const config = { AUTH0_INPUT_FILE: mainFile }; + const context = new Context(config, mockMgmtClient()); + + await context.loadAssetsFromLocal(); + + expect(context.assets.tenant.friendly_name).to.equal('Test'); + expect(context.assets.tenant.data.shared_value).to.equal(42); + expect(context.assets.rules).to.deep.equal([]); + }); + + it('should allow same file included multiple times in different branches', async () => { + const dir = path.resolve(testDataDir, 'yaml', 'multiple-includes'); + cleanThenMkdir(dir); + + const mainFile = path.join(dir, 'main.yaml'); + fs.writeFileSync( + mainFile, + ` +tenant: + friendly_name: Test + config1: !include shared.yaml + config2: !include shared.yaml +` + ); + + const sharedFile = path.join(dir, 'shared.yaml'); + fs.writeFileSync(sharedFile, 'shared'); + + const config = { AUTH0_INPUT_FILE: mainFile }; + const context = new Context(config, mockMgmtClient()); + + await context.loadAssetsFromLocal(); + + expect(context.assets.tenant.friendly_name).to.equal('Test'); + expect(context.assets.tenant.config1).to.equal('shared'); + expect(context.assets.tenant.config2).to.equal('shared'); + }); +});