Merge branch 'main' into Issue-1354

Author: niamhg-ms

shared/src/docify/docifier.e2e.spec.ts

@@ -1,24 +1,21 @@
-import { describe, it, expect, afterEach } from 'vitest';
+import { describe, it, afterEach } from 'vitest';
 import { Docifier } from './docifier.js';
-import { rmSync, existsSync} from 'fs';
+import { rmSync} from 'fs';
 import { join } from 'path';
+import {expectDirectoryMatch} from '../test/file-comparison';
 
 const INPUT_DIR = join(
     __dirname,
     '../../test_fixtures/command/generate/expected-output'
 );
-
 const WORKSHOP_DIR = join(
     __dirname,
     '../../../calm/workshop/controls'
 );
 
-const CALM_DIR = join(
-    __dirname,
-    '../../../calm/release/1.0-rc1/meta'
-);
-
 const OUTPUT_DIR = join(__dirname, '../../test_fixtures/docify/workshop/actual-output');
+const EXPECTED_OUTPUT_DIR = join(__dirname, '../../test_fixtures/docify/workshop/expected-output');
+
 const NON_SECURE_VERSION_DOC_WEBSITE =  join(OUTPUT_DIR,'non-secure');
 const SECURE_VERSION_DOC_WEBSITE =  join(OUTPUT_DIR,'secure');
 
@@ -28,44 +25,29 @@ describe('Docifier E2E - Real Model and Template', () => {
     });
 
     it('generates documentation from the conference-signup.arch.json model', async () => {
-        const mapping = new Map<string, string>();
 
-        const docifier = new Docifier('WEBSITE', join(INPUT_DIR, 'conference-signup.arch.json'), NON_SECURE_VERSION_DOC_WEBSITE, mapping);
+        const docifier = new Docifier('WEBSITE', join(INPUT_DIR, 'conference-signup.arch.json'), NON_SECURE_VERSION_DOC_WEBSITE, new Map<string,string>());
         await docifier.docify();
-
-        //Verifying a few files
-        const packageJsonPath = join(NON_SECURE_VERSION_DOC_WEBSITE, 'package.json');
-        const indexMdPath = join(NON_SECURE_VERSION_DOC_WEBSITE, 'docs/index.md');
-
-        expect(existsSync(NON_SECURE_VERSION_DOC_WEBSITE)).toBe(true);
-        expect(existsSync(packageJsonPath)).toBe(true);
-        expect(existsSync(indexMdPath)).toBe(true);
+        await expectDirectoryMatch(join(EXPECTED_OUTPUT_DIR,'non-secure'),join(OUTPUT_DIR,'non-secure'));
 
     });
 
-    it('generates documentation from the conference-secure-signup.arch.json model', async () => {
+    it('generates documentation from the conference-secure-signup.arch.json model with explicit local mapping', async () => {
         const mapping = new Map<string, string>([
-            ['https://calm.finos.org/release/1.0-rc1/meta/control-requirement.json', join(CALM_DIR, 'control-requirement.json')],
-            ['https://calm.finos.org/workshop/controls/micro-segmentation.config.json', join(WORKSHOP_DIR, 'micro-segmentation.config.json')],
-            ['https://calm.finos.org/workshop/controls/permitted-connection-http.config.json', join(WORKSHOP_DIR, 'permitted-connection-http.config.json')],
-            ['https://calm.finos.org/workshop/controls/permitted-connection-jdbc.config.json', join(WORKSHOP_DIR, 'permitted-connection-jdbc.config.json')],
             ['https://calm.finos.org/workshop/controls/micro-segmentation.config.json', join(WORKSHOP_DIR, 'micro-segmentation.config.json')],
-            ['https://calm.finos.org/workshop/controls/micro-segmentation.requirement.json', join(WORKSHOP_DIR, 'micro-segmentation.requirement.json')],
-            ['https://calm.finos.org/workshop/controls/permitted-connection.requirement.json', join(WORKSHOP_DIR, 'permitted-connection.requirement.json')],
         ]);
 
-
         const docifier = new Docifier('WEBSITE', join(INPUT_DIR, 'conference-secure-signup-amended.arch.json'), SECURE_VERSION_DOC_WEBSITE, mapping);
 
         await docifier.docify();
+        await expectDirectoryMatch(join(EXPECTED_OUTPUT_DIR,'secure'),join(OUTPUT_DIR,'secure'));
 
-        //Verifying a few files
-        const packageJsonPath = join(SECURE_VERSION_DOC_WEBSITE, 'package.json');
-        const indexMdPath = join(SECURE_VERSION_DOC_WEBSITE, 'docs/index.md');
+    });
 
-        expect(existsSync(SECURE_VERSION_DOC_WEBSITE)).toBe(true);
-        expect(existsSync(packageJsonPath)).toBe(true);
-        expect(existsSync(indexMdPath)).toBe(true);
+    it('generates documentation from the conference-secure-signup.arch.json model with no mapping as workshop documents are available', async () => {
+        const docifier = new Docifier('WEBSITE', join(INPUT_DIR, 'conference-secure-signup-amended.arch.json'), SECURE_VERSION_DOC_WEBSITE, new Map<string,string>());
+        await docifier.docify();
+        await expectDirectoryMatch(join(EXPECTED_OUTPUT_DIR,'secure'),join(OUTPUT_DIR,'secure'));
 
     });
-});
+});

shared/src/docify/graphing/control-registry.ts

@@ -1,4 +1,3 @@
-/* eslint-disable  @typescript-eslint/no-explicit-any */
 import { Architecture } from '../../model/core';
 import { CalmControl } from '../../model/control';
 

shared/src/docify/graphing/flow-sequence-helper.ts

@@ -1,4 +1,3 @@
-/* eslint-disable  @typescript-eslint/no-explicit-any */
 import { Architecture } from '../../model/core';
 import { CalmFlowTransition } from '../../model/flow';
 import { CalmRelationship, CalmInteractsType, CalmConnectsType, CalmComposedOfType } from '../../model/relationship';

shared/src/resolver/network-addressable-extractor.e2e.spec.ts

@@ -0,0 +1,63 @@
+import { readFileSync } from 'fs';
+import { join } from 'path';
+import { describe, it, expect } from 'vitest';
+import { extractNetworkAddressables, AddressableEntry } from './network-addressable-extractor.js';
+
+const WORKSHOP_DIR = join(__dirname, '../../../calm/workshop');
+const jsonPath = join(WORKSHOP_DIR, 'architecture/conference-secure-signup-amended.arch.json');
+const jsonDoc = readFileSync(jsonPath, 'utf-8');
+const entries: AddressableEntry[] = extractNetworkAddressables(jsonDoc);
+
+describe('extractNetworkAddressables E2E', () => {
+    it('extracts the web URL from conference-website interface', () => {
+        expect(entries).toContainEqual({
+            path: 'root.nodes[0].interfaces[0].url',
+            key: 'url',
+            value: 'https://calm.finos.org/amazing-website'
+        });
+    });
+
+    it('extracts control requirement URLs from node controls', () => {
+        expect(entries).toContainEqual({
+            path: 'root.nodes[4].controls.security.requirements[0].control-requirement-url',
+            key: 'control-requirement-url',
+            value: 'https://calm.finos.org/workshop/controls/micro-segmentation.requirement.json'
+        });
+    });
+
+    it('extracts control config URLs from node controls', () => {
+        expect(entries).toContainEqual({
+            path: 'root.nodes[4].controls.security.requirements[0].control-config-url',
+            key: 'control-config-url',
+            value: 'https://calm.finos.org/workshop/controls/micro-segmentation.config.json'
+        });
+    });
+
+    it('extracts requirement URLs from relationships controls', () => {
+        const relReqs = entries
+            .filter(e => e.key === 'control-requirement-url' && e.path.startsWith('root.relationships'))
+            .map(e => e.value)
+            .sort();
+        expect(relReqs).toEqual([
+            'https://calm.finos.org/workshop/controls/permitted-connection.requirement.json',
+            'https://calm.finos.org/workshop/controls/permitted-connection.requirement.json',
+            'https://calm.finos.org/workshop/controls/permitted-connection.requirement.json'
+        ].sort());
+    });
+
+    it('extracts config URLs from relationships controls', () => {
+        const relConfigs = entries
+            .filter(e => e.key === 'control-config-url' && e.path.startsWith('root.relationships'))
+            .map(e => e.value)
+            .sort();
+        expect(relConfigs).toEqual([
+            'https://calm.finos.org/workshop/controls/permitted-connection-http.config.json',
+            'https://calm.finos.org/workshop/controls/permitted-connection-http.config.json',
+            'https://calm.finos.org/workshop/controls/permitted-connection-jdbc.config.json'
+        ].sort());
+    });
+
+    it('does not extract non-URL values', () => {
+        expect(entries.every(e => /^https?:\/\//.test(e.value))).toBe(true);
+    });
+});

shared/src/resolver/network-addressable-extractor.spec.ts

@@ -0,0 +1,48 @@
+import { describe, it, expect } from 'vitest';
+import { extractNetworkAddressables } from './network-addressable-extractor.js';
+
+describe('extractNetworkAddressables', () => {
+    it('extracts URLs from string values', () => {
+        const json = JSON.stringify({
+            website: 'http://example.com',
+            api: 'https://api.example.org/v1'
+        });
+        const entries = extractNetworkAddressables(json);
+        expect(entries).toEqual(
+            expect.arrayContaining([
+                { path: 'root.website', key: 'website', value: 'http://example.com' },
+                { path: 'root.api', key: 'api', value: 'https://api.example.org/v1' }
+            ])
+        );
+    });
+
+    it('extracts URLs from nested objects and arrays', () => {
+        const doc = {
+            items: [
+                { link: 'http://foo.com' },
+                ['https://bar.com', { deep: 'http://deep.example.com' }]
+            ],
+            'https://in-key.com': 'value'
+        };
+        const json = JSON.stringify(doc);
+        const entries = extractNetworkAddressables(json);
+        expect(entries).toEqual(
+            expect.arrayContaining([
+                { path: 'root.items[0].link', key: 'link', value: 'http://foo.com' },
+                { path: 'root.items[1][0]', key: '0', value: 'https://bar.com' },
+                { path: 'root.items[1][1].deep', key: 'deep', value: 'http://deep.example.com' },
+                { path: 'root.https://in-key.com', key: 'https://in-key.com', value: 'https://in-key.com' }
+            ])
+        );
+    });
+
+    it('throws on invalid JSON input', () => {
+        expect(() => extractNetworkAddressables('not a json')).toThrow(/Invalid JSON string provided/);
+    });
+
+    it('does not extract when no URLs present', () => {
+        const json = JSON.stringify({ a: 1, b: ['x', { c: 'y' }] });
+        const entries = extractNetworkAddressables(json);
+        expect(entries).toEqual([]);
+    });
+});

shared/src/resolver/network-addressable-extractor.ts

@@ -0,0 +1,54 @@
+export interface AddressableEntry {
+    path: string;
+    key: string;
+    value: string;
+}
+
+const URL_REGEX = /https?:\/\/[^\s"']+/g;
+
+export function extractNetworkAddressables(jsonString: string): AddressableEntry[] {
+    let data: unknown;
+    try {
+        data = JSON.parse(jsonString);
+    } catch (err) {
+        throw new Error(`Invalid JSON string provided: ${err.message}`);
+    }
+    const results: AddressableEntry[] = [];
+    traverse(data, 'root', results);
+    return results;
+}
+
+function traverse(current: unknown, path: string, results: AddressableEntry[]): void {
+    if (typeof current === 'string') {
+        pushUrls(path, extractKey(path), current, results);
+        return;
+    }
+
+    if (Array.isArray(current)) {
+        current.forEach((item, idx) => traverse(item, `${path}[${idx}]`, results));
+        return;
+    }
+
+    if (current && typeof current === 'object') {
+        const obj = current as Record<string, unknown>;
+        for (const [key, value] of Object.entries(obj)) {
+            const keyPath = `${path}.${key}`;
+            pushUrls(keyPath, key, key, results);
+            if (typeof value === 'string') {
+                pushUrls(keyPath, key, value, results);
+            } else if (value) {
+                traverse(value, keyPath, results);
+            }
+        }
+    }
+}
+
+function extractKey(path: string): string {
+    const match = path.match(/(?:\.([^.[\]]+)|\[(\d+)])$/);
+    return match ? (match[1] ?? match[2]!) : '';
+}
+
+function pushUrls(path: string, key: string, text: string, results: AddressableEntry[]) {
+    const matches = text.match(URL_REGEX);
+    matches?.forEach(url => results.push({ path, key, value: url }));
+}

shared/src/resolver/network-addressable-validator.e2e.spec.ts

@@ -0,0 +1,65 @@
+import { readFileSync } from 'fs';
+import { join } from 'path';
+import { describe, it, expect } from 'vitest';
+import { extractNetworkAddressables, AddressableEntry } from './network-addressable-extractor.js';
+import { NetworkAddressableValidator, ValidationResult } from './network-addressable-validator.js';
+
+const WORKSHOP_DIR = join(__dirname, '../../../calm/workshop');
+const jsonPath = join(
+    WORKSHOP_DIR,
+    'architecture/conference-secure-signup-amended.arch.json'
+);
+const jsonDoc = readFileSync(jsonPath, 'utf-8');
+const entries: AddressableEntry[] = extractNetworkAddressables(jsonDoc);
+
+// Using default HttpReferenceResolver; ensure network access is available
+const validator = new NetworkAddressableValidator();
+
+describe('NetworkAddressableValidator E2E', () => {
+    let results: ValidationResult[];
+
+    beforeAll(async () => {
+        results = await validator.validate(entries);
+    });
+
+    it('validates the conference website URL as reachable and not a schema', () => {
+        const entry = entries.find(
+            e => e.value === 'https://calm.finos.org/amazing-website'
+        );
+        const result = results.find(r => r.entry === entry);
+        expect(result).toBeDefined();
+        expect(result?.reachable).toBe(false);
+        expect(result?.isSchemaDefinition).toBe(false);
+        expect(result?.isSchemaImplementation).toBe(false);
+        expect(result?.error).toBe('HTTP request failed for https://calm.finos.org/amazing-website: Request failed with status code 404');
+    });
+
+    it('validates a control-requirement URL as schema document', () => {
+        const entry = entries.find(
+            e => e.key === 'control-requirement-url'
+        );
+        const result = results.find(r => r.entry === entry);
+        expect(result).toBeDefined();
+        expect(result?.reachable).toBe(true);
+        expect(result?.isSchemaDefinition).toBe(true);
+        expect(result?.isSchemaImplementation).toBe(false);
+    });
+
+    it('validates a control-config URL as schema document', () => {
+        const entry = entries.find(
+            e => e.key === 'control-config-url'
+        );
+        const result = results.find(r => r.entry === entry);
+        expect(result).toBeDefined();
+        expect(result?.reachable).toBe(true);
+        expect(result?.isSchemaDefinition).toBe(false);
+        expect(result?.isSchemaImplementation).toBe(true);
+    });
+
+    it('all entries have a reachable or error result', () => {
+        expect(results.length).toBe(entries.length);
+        results.forEach(r => {
+            expect(r.reachable || r.error).toBeDefined();
+        });
+    });
+});