feat: add support for YAML file includes and nested includes in context

src/context/yaml/index.ts

@@ -18,6 +18,79 @@ import { Assets, Config, Auth0APIClient, AssetTypes, KeywordMappings } from '../
 import { filterOnlyIncludedResourceTypes } from '..';
 import { preserveKeywords } from '../../keywordPreservation';
 
+// Custom YAML type for file includes
+const includeType = new yaml.Type('!include', {
+  kind: 'scalar',
+  resolve: (data) => typeof data === 'string',
+  construct: (data) => {
+    // This will be handled during the actual loading process
+    return { __include: data };
+  },
+});
+
+const schema = yaml.DEFAULT_SCHEMA.extend([includeType]);
+
+function prepareYamlContent(
+  content: string,
+  mappings: KeywordMappings,
+  disableKeywordReplacement: boolean
+): string {
+  return disableKeywordReplacement
+    ? wrapArrayReplaceMarkersInQuotes(content, mappings)
+    : keywordReplace(content, mappings);
+}
+
+function parseYaml(content: string) {
+  return yaml.load(content, { schema });
+}
+
+type LoadIncludedYaml = (filePath: string) => any;
+
+// Resolves !include directives only. Keyword handling is applied by the caller
+// when preparing each file's raw content before parse.
+function resolveIncludes(
+  obj: any,
+  basePath: string,
+  loadIncludedYaml: LoadIncludedYaml,
+  visitedFiles = new Set<string>()
+): any {
+  if (Array.isArray(obj)) {
+    return obj.map((item) => resolveIncludes(item, basePath, loadIncludedYaml, visitedFiles));
+  }
+
+  if (obj && typeof obj === 'object') {
+    if (obj.__include) {
+      const filePath = path.resolve(basePath, obj.__include);
+
+      if (visitedFiles.has(filePath)) {
+        throw new Error(`Circular include detected: ${filePath}`);
+      }
+
+      if (!fs.existsSync(filePath)) {
+        throw new Error(`Include file not found: ${filePath}`);
+      }
+
+      visitedFiles.add(filePath);
+      const result = resolveIncludes(
+        loadIncludedYaml(filePath),
+        path.dirname(filePath),
+        loadIncludedYaml,
+        new Set(visitedFiles)
+      );
+      visitedFiles.delete(filePath);
+      return result;
+    }
+
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+      result[key] = resolveIncludes(value, basePath, loadIncludedYaml, visitedFiles);
+    }
+    return result;
+  }
+
+  return obj;
+}
+
 export default class YAMLContext {
   basePath: string;
   configFile: string;
@@ -83,13 +156,18 @@ export default class YAMLContext {
       try {
         const fPath = path.resolve(this.configFile);
         log.debug(`Loading YAML from ${fPath}`);
+
+        const prepareContent = (content: string) =>
+          prepareYamlContent(content, this.mappings, opts.disableKeywordReplacement);
+
+        const loadIncludedYaml: LoadIncludedYaml = (filePath) =>
+          parseYaml(prepareContent(fs.readFileSync(filePath, 'utf8')));
+
+        const parsed = parseYaml(prepareContent(fs.readFileSync(fPath, 'utf8')));
+
         Object.assign(
           this.assets,
-          yaml.load(
-            opts.disableKeywordReplacement
-              ? wrapArrayReplaceMarkersInQuotes(fs.readFileSync(fPath, 'utf8'), this.mappings)
-              : keywordReplace(fs.readFileSync(fPath, 'utf8'), this.mappings)
-          ) || {}
+          resolveIncludes(parsed, path.dirname(fPath), loadIncludedYaml) || {}
         );
       } catch (err) {
         log.debug(err.stack);

test/context/yaml/actions.test.js

@@ -66,6 +66,169 @@ describe('#YAML context actions', () => {
     expect(context.assets.actions).to.deep.equal(target);
   });
 
+  it('should process YAML with includes', async () => {
+    const dir = path.join(testDataDir, 'yaml', 'includes');
+    cleanThenMkdir(dir);
+
+    const clientsYaml = `
+- name: "Test Client"
+  app_type: "spa"
+- name: "Test M2M"
+  app_type: "non_interactive"
+    `;
+    const clientsFile = path.join(dir, 'clients.yaml');
+    fs.writeFileSync(clientsFile, clientsYaml);
+
+    const mainYaml = `
+tenant:
+  friendly_name: 'Test Tenant'
+
+clients: !include clients.yaml
+    `;
+    const mainFile = path.join(dir, 'tenant.yaml');
+    fs.writeFileSync(mainFile, mainYaml);
+
+    const config = { AUTH0_INPUT_FILE: mainFile };
+    const context = new Context(config, mockMgmtClient());
+    await context.loadAssetsFromLocal();
+
+    expect(context.assets.tenant).to.deep.equal({
+      friendly_name: 'Test Tenant',
+    });
+    expect(context.assets.clients).to.deep.equal([
+      {
+        name: 'Test Client',
+        app_type: 'spa',
+      },
+      {
+        name: 'Test M2M',
+        app_type: 'non_interactive',
+      },
+    ]);
+  });
+
+  it('should handle nested includes', async () => {
+    const dir = path.join(testDataDir, 'yaml', 'nested-includes');
+    cleanThenMkdir(dir);
+
+    const rolesYaml = `
+- name: Admin
+  description: Administrator
+- name: User
+  description: Regular User
+    `;
+    fs.writeFileSync(path.join(dir, 'roles.yaml'), rolesYaml);
+
+    const mainYaml = `
+tenant:
+  friendly_name: 'Main Tenant'
+
+roles: !include roles.yaml
+    `;
+    fs.writeFileSync(path.join(dir, 'tenant.yaml'), mainYaml);
+
+    const config = { AUTH0_INPUT_FILE: path.join(dir, 'tenant.yaml') };
+    const context = new Context(config, mockMgmtClient());
+    await context.loadAssetsFromLocal();
+
+    expect(context.assets.roles).to.deep.equal([
+      { name: 'Admin', description: 'Administrator' },
+      { name: 'User', description: 'Regular User' },
+    ]);
+  });
+
+  it('should process logStreams with includes', async () => {
+    const dir = path.join(testDataDir, 'yaml', 'logstreams-includes');
+    cleanThenMkdir(dir);
+
+    const logStreamsYaml = `
+- name: LoggingSAAS
+  isPriority: false
+  filters:
+  - type: category
+    name: auth.login.fail
+  - type: category
+    name: auth.login.notification
+  - type: category
+    name: auth.login.success
+  - type: category
+    name: auth.logout.fail
+  sink:
+    httpContentFormat: JSONLINES
+    httpContentType: application/json
+    httpEndpoint: "##LOGGING_WEBHOOK_URL##"
+  type: http
+- name: SIEM
+  isPriority: false
+  filters:
+  - type: category
+    name: auth.login.fail
+  - type: category
+    name: auth.login.notification
+  - type: category
+    name: auth.login.success
+  sink:
+    httpContentFormat: JSONLINES
+    httpContentType: application/json
+    httpEndpoint: "##SIEM_WEBHOOK_URL##"
+  type: http
+    `;
+    fs.writeFileSync(path.join(dir, 'logStreams.yaml'), logStreamsYaml);
+
+    const mainYaml = `
+tenant:
+  friendly_name: 'Test Tenant'
+
+logStreams: !include logStreams.yaml
+    `;
+    fs.writeFileSync(path.join(dir, 'tenant.yaml'), mainYaml);
+
+    const config = {
+      AUTH0_INPUT_FILE: path.join(dir, 'tenant.yaml'),
+      AUTH0_KEYWORD_REPLACE_MAPPINGS: {
+        LOGGING_WEBHOOK_URL: 'https://logging.com/inputs/test',
+        SIEM_WEBHOOK_URL: 'https://siem.example.com/webhook',
+      },
+    };
+    const context = new Context(config, mockMgmtClient());
+    await context.loadAssetsFromLocal();
+
+    expect(context.assets.logStreams).to.have.length(2);
+    expect(context.assets.logStreams[0]).to.deep.include({
+      name: 'LoggingSAAS',
+      isPriority: false,
+      type: 'http',
+    });
+    expect(context.assets.logStreams[0].sink.httpEndpoint).to.equal(
+      'https://logging.com/inputs/test'
+    );
+    expect(context.assets.logStreams[1]).to.deep.include({
+      name: 'SIEM',
+      isPriority: false,
+      type: 'http',
+    });
+    expect(context.assets.logStreams[1].sink.httpEndpoint).to.equal(
+      'https://siem.example.com/webhook'
+    );
+  });
+
+  it('should error on missing include file', async () => {
+    const dir = path.join(testDataDir, 'yaml', 'missing-include');
+    cleanThenMkdir(dir);
+
+    const mainYaml = `
+clients: !include missing.yaml
+    `;
+    fs.writeFileSync(path.join(dir, 'tenant.yaml'), mainYaml);
+
+    const config = { AUTH0_INPUT_FILE: path.join(dir, 'tenant.yaml') };
+    const context = new Context(config, mockMgmtClient());
+
+    await expect(context.loadAssetsFromLocal()).to.be.eventually.rejectedWith(
+      Error,
+      /Include file not found/
+    );
+  });
   it('should dump actions', async () => {
     const dir = path.join(testDataDir, 'yaml', 'actionsDump');
     cleanThenMkdir(dir);