diff --git a/cli/cli-auth-plugins/calmhub-curl.sh b/cli/cli-auth-plugins/calmhub-curl.sh new file mode 100755 index 000000000..f737ed4c8 --- /dev/null +++ b/cli/cli-auth-plugins/calmhub-curl.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Simple wrapper around curl that offers Kerberos authentication (--negotiate -u:) +# Usage: calmhub-curl.sh [-m METHOD] URL +# If METHOD is POST, the request body is read from stdin. + +show_help() { + cat <&2 + exit 1 + fi + ;; + -h|--help) + show_help + exit 0 + ;; + *) + url="$1" + shift + ;; + esac +done + +if [[ -z "$url" ]]; then + echo "Error: URL is required." >&2 + show_help + exit 1 +fi + +# Normalize method to uppercase +method=$(echo "$method" | tr '[:lower:]' '[:upper:]') + +COOKIE_JAR=/var/tmp/calmhub_cookies.$(id -un) +CURL_OPTS="--silent --fail-with-body --location-trusted --negotiate -u: -b $COOKIE_JAR -c $COOKIE_JAR -w %{stderr}%{http_code}" + +# Ensure cookie jar file exists with secure permissions (current user ONLY) +old_umask=$(umask) +umask 0077 +touch "$COOKIE_JAR" +umask "$old_umask" + +# Execute curl. For POST, read body from stdin. +if [[ "$method" == "POST" ]]; then + if [ -t 0 ]; then + # No stdin data; send empty payload + curl ${CURL_OPTS} -X "$method" -H "Content-Type: application/json" --data-binary @- "$url" < /dev/null + exit $? + else + # Pipe actual stdin to curl + curl ${CURL_OPTS} -X "$method" -H "Content-Type: application/json" --data-binary @- "$url" + exit $? + fi +else + # Methods without body + curl ${CURL_OPTS} "$url" + exit $? +fi diff --git a/cli/package.json b/cli/package.json index 096ac05a6..02f19437a 100644 --- a/cli/package.json +++ b/cli/package.json @@ -15,12 +15,13 @@ "calm": "dist/index.js" }, "scripts": { - "build": "tsup && npm run copy-calm-schema && npm run copy-docify-templates && npm run copy-widgets && npm run copy-ai-tools", + "build": "tsup && npm run copy-calm-schema && npm run copy-docify-templates && npm run copy-widgets && npm run copy-ai-tools && npm run copy-auth-plugins", "watch": "node watch.mjs", "copy-calm-schema": "copyfiles \"../calm/release/**/meta/*\" \"../calm/draft/**/meta/*\" dist/calm/", "copy-docify-templates": "copyfiles \"../shared/dist/template-bundles/**/*\" dist --up 3", "copy-widgets": "copyfiles \"../calm-widgets/dist/cli/widgets/**/*\" dist --up 4", "copy-ai-tools": "copyfiles \"../calm-ai/tools/**/*\" \"../calm-ai/CALM.chatmode.md\" dist/calm-ai/ --up 2", + "copy-auth-plugins": "copyfiles \"./cli-auth-plugins/**/*\" dist/plugins/ --up 1", "test": "vitest run", "lint": "eslint src", "lint-fix": "eslint src --fix", diff --git a/cli/src/cli-config.spec.ts b/cli/src/cli-config.spec.ts index 286f63894..87dc32e30 100644 --- a/cli/src/cli-config.spec.ts +++ b/cli/src/cli-config.spec.ts @@ -15,6 +15,9 @@ const exampleConfig = { calmHubUrl: 'https://example.com/calmhub' }; +const exampleConfig2 = { + calmHubPlugin: '/my/plugin/script' +}; describe('cli-config', () => { beforeEach(() => { @@ -33,6 +36,14 @@ describe('cli-config', () => { expect(config).toEqual(exampleConfig); }); + it('loads second user config from .calm.json in home dir', async () => { + vol.fromJSON({ + '/home/user/.calm.json': JSON.stringify(exampleConfig2) + }); + const config = await loadCliConfig(); + expect(config).toEqual(exampleConfig2); + }); + it('returns null when .calm.json does not exist', async () => { const config = await loadCliConfig(); expect(config).toBeNull(); diff --git a/cli/src/cli-config.ts b/cli/src/cli-config.ts index 35e43bec3..b544f4f1d 100644 --- a/cli/src/cli-config.ts +++ b/cli/src/cli-config.ts @@ -4,7 +4,8 @@ import { homedir } from 'os'; import { join } from 'path'; export interface CLIConfig { - calmHubUrl?: string + calmHubUrl?: string, + calmHubPlugin?: string } function getUserConfigLocation(): string { diff --git a/cli/src/cli.spec.ts b/cli/src/cli.spec.ts index 1e030bde9..b070c735b 100644 --- a/cli/src/cli.spec.ts +++ b/cli/src/cli.spec.ts @@ -14,7 +14,9 @@ let serverModule: typeof import('./server/cli-server'); let templateModule: typeof import('./command-helpers/template'); let optionsModule: typeof import('./command-helpers/generate-options'); let fileSystemDocLoaderModule: typeof import('@finos/calm-shared/dist/document-loader/file-system-document-loader'); +let cliConfigModule: typeof import('./cli-config'); let setupCLI: typeof import('./cli').setupCLI; +let parseDocumentLoaderConfig: typeof import('./cli').parseDocumentLoaderConfig; describe('CLI Commands', () => { let program: Command; @@ -22,6 +24,7 @@ describe('CLI Commands', () => { beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); + vi.unstubAllEnvs(); calmShared = await import('@finos/calm-shared'); validateModule = await import('./command-helpers/validate'); @@ -29,6 +32,7 @@ describe('CLI Commands', () => { templateModule = await import('./command-helpers/template'); optionsModule = await import('./command-helpers/generate-options'); fileSystemDocLoaderModule = await import('@finos/calm-shared/dist/document-loader/file-system-document-loader'); + cliConfigModule = await import('./cli-config'); vi.spyOn(calmShared, 'runGenerate').mockResolvedValue(undefined); vi.spyOn(calmShared.TemplateProcessor.prototype, 'processTemplate').mockResolvedValue(undefined); @@ -45,8 +49,11 @@ describe('CLI Commands', () => { vi.spyOn(fileSystemDocLoaderModule, 'FileSystemDocumentLoader').mockImplementation(vi.fn()); vi.spyOn(fileSystemDocLoaderModule.FileSystemDocumentLoader.prototype, 'loadMissingDocument').mockResolvedValue({}); + vi.spyOn(cliConfigModule, 'loadCliConfig').mockImplementation(vi.fn()); + const cliModule = await import('./cli'); setupCLI = cliModule.setupCLI; + parseDocumentLoaderConfig = cliModule.parseDocumentLoaderConfig; program = new Command(); setupCLI(program); @@ -337,4 +344,69 @@ describe('CLI Commands', () => { }); }); + describe('Document Loader options', () => { + it('config when no options selected', async () => { + const config = await parseDocumentLoaderConfig({ + }); + expect(config.calmHubUrl).toBeUndefined(); + expect(config.calmHubPlugin).toBeUndefined(); + expect(config.schemaDirectoryPath).toBeUndefined(); + expect(config.debug).toBeFalsy(); + }); + + it('config with CalmHub defined in config file', async () => { + vi.spyOn(cliConfigModule, 'loadCliConfig').mockImplementation(() => { + return { + calmHubUrl: 'calmhub.local', + calmHubPlugin: 'plugin-name' + }; + }); + + const config = await parseDocumentLoaderConfig({}); + expect(config.calmHubUrl).toBe('calmhub.local'); + expect(config.calmHubPlugin).toBe('plugin-name'); + }); + + it('config with CalmHub defined in config file overridden by options', async () => { + vi.spyOn(cliConfigModule, 'loadCliConfig').mockImplementation(() => { + return { + calmHubUrl: 'calmhub.local', + calmHubPlugin: 'plugin-name' + }; + }); + + const config = await parseDocumentLoaderConfig({ + calmHubUrl: 'override.local', + calmHubPlugin: 'override-plugin' + }); + expect(config.calmHubUrl).toBe('override.local'); + expect(config.calmHubPlugin).toBe('override-plugin'); + }); + + it('config with CalmHub defined in config file overridden by environment', async () => { + vi.spyOn(cliConfigModule, 'loadCliConfig').mockImplementation(() => { + return { + calmHubUrl: 'calmhub.local', + calmHubPlugin: 'plugin-name' + }; + }); + + vi.stubEnv('CALM_HUB_URL', 'env.local'); + vi.stubEnv('CALM_HUB_PLUGIN', 'env-plugin'); + const config = await parseDocumentLoaderConfig({}); + expect(config.calmHubUrl).toBe('env.local'); + expect(config.calmHubPlugin).toBe('env-plugin'); + }); + + it('config with CalmHub defined in environment overridden by options', async () => { + vi.stubEnv('CALM_HUB_URL', 'env.local'); + vi.stubEnv('CALM_HUB_PLUGIN', 'env-plugin'); + const config = await parseDocumentLoaderConfig({ + calmHubUrl: 'calmhub.local', + calmHubPlugin: 'plugin-name' + }); + expect(config.calmHubUrl).toBe('calmhub.local'); + expect(config.calmHubPlugin).toBe('plugin-name'); + }); + }); }); diff --git a/cli/src/cli.ts b/cli/src/cli.ts index c78a20e3b..21872374e 100644 --- a/cli/src/cli.ts +++ b/cli/src/cli.ts @@ -15,6 +15,7 @@ const VERBOSE_OPTION = '-v, --verbose'; // Generate command options const PATTERN_OPTION = '-p, --pattern '; const CALMHUB_URL_OPTION = '-c, --calm-hub-url '; +const CALMHUB_PLUGIN_OPTION = '--calm-hub-plugin '; // Validate command options const FORMAT_OPTION = '-f, --format '; @@ -43,6 +44,7 @@ export function setupCLI(program: Command) { .requiredOption(OUTPUT_OPTION, 'Path location at which to output the generated file.', 'architecture.json') .option(SCHEMAS_OPTION, 'Path to the directory containing the meta schemas to use.') .option(CALMHUB_URL_OPTION, 'URL to CALMHub instance') + .option(CALMHUB_PLUGIN_OPTION, 'Plugin to support custom CALMHub access') .option(VERBOSE_OPTION, 'Enable verbose logging.', false) .action(async (options) => { const debug = !!options.verbose; @@ -67,6 +69,8 @@ export function setupCLI(program: Command) { .default('json') ) .option(OUTPUT_OPTION, 'Path location at which to output the generated file.') + .option(CALMHUB_URL_OPTION, 'URL to CALMHub instance') + .option(CALMHUB_PLUGIN_OPTION, 'Plugin to support custom CALMHub access') .option(VERBOSE_OPTION, 'Enable verbose logging.', false) .action(async (options) => { const { checkValidateOptions, runValidate } = await import('./command-helpers/validate'); @@ -78,7 +82,9 @@ export function setupCLI(program: Command) { verbose: !!options.verbose, strict: options.strict, outputFormat: options.format, - outputPath: options.output + outputPath: options.output, + calmHubUrl: options.calmHubUrl, + calmHubPlugin: options.calmHubPlugin }); }); @@ -87,6 +93,8 @@ export function setupCLI(program: Command) { .description('Start a HTTP server to proxy CLI commands. (experimental)') .option(PORT_OPTION, 'Port to run the server on', '3000') .requiredOption(SCHEMAS_OPTION, 'Path to the directory containing the meta schemas to use.') + .option(CALMHUB_URL_OPTION, 'URL to CALMHub instance') + .option(CALMHUB_PLUGIN_OPTION, 'Plugin to support custom CALMHub access') .option(VERBOSE_OPTION, 'Enable verbose logging.', false) .action(async (options) => { const { startServer } = await import('./server/cli-server'); @@ -223,15 +231,39 @@ export async function parseDocumentLoaderConfig(options): Promise ({ + execFileSync: vi.fn() +})); +const execFileSyncMock = (execFileSync as Mock); + +describe('calmhub-custom-document-loader', () => { + let calmHubDocumentLoader: CalmHubCustomDocumentLoader; + let schemaDirectory: SchemaDirectory; + beforeEach(() => { + calmHubDocumentLoader = new CalmHubCustomDocumentLoader(calmHubBaseUrl, 'my-calmhub-wrapper', false); + calmHubDocumentLoader.initialise(schemaDirectory); + execFileSyncMock.mockClear(); + }); + + it('loads a document from CalmHub', async () => { + const calmHubUrl = 'calm:/schemas/2025-03/meta/core.json'; + + const mockResponse = JSON.stringify({ + '$id': 'https://calm.finos.org/calm/schemas/2025-03/meta/core.json', + 'value': 'test' + }); + + execFileSyncMock.mockReturnValueOnce(mockResponse); + + const document = await calmHubDocumentLoader.loadMissingDocument(calmHubUrl, 'schema'); + expect(document).toEqual(JSON.parse(mockResponse)); + expect(execFileSyncMock).toHaveBeenCalledExactlyOnceWith( + path.resolve(CALM_AUTH_PLUGIN_DIRECTORY, 'my-calmhub-wrapper'), + ['--method', 'GET', calmHubBaseUrl + '/schemas/2025-03/meta/core.json'], + { 'stdio': 'pipe', 'shell': true, 'encoding': 'utf-8', 'timeout': 30000 }); + }); + + it('throws an error when the document is not found', async () => { + const calmHubUrl = 'calm:/schemas/2025-03/meta/nonexistent.json'; + + execFileSyncMock.mockImplementation(() => { + throw { + status: 404, + stderr: '

Not Found

' + }; + }); + + await expect(calmHubDocumentLoader.loadMissingDocument(calmHubUrl, 'schema')).rejects.toThrow(); + expect(execFileSyncMock).toHaveBeenCalledExactlyOnceWith( + path.resolve(CALM_AUTH_PLUGIN_DIRECTORY, 'my-calmhub-wrapper'), + ['--method', 'GET', calmHubBaseUrl + '/schemas/2025-03/meta/nonexistent.json'], + { 'stdio': 'pipe', 'shell': true, 'encoding': 'utf-8', 'timeout': 30000 }); + }); + + it('throws an error when the wrapper fails:', async () => { + const calmHubUrl = 'calm:/schemas/2025-03/meta/nonexistent.json'; + + execFileSyncMock.mockImplementation(() => { + throw { + code: constants.errno.ENOENT + }; + }); + + await expect(calmHubDocumentLoader.loadMissingDocument(calmHubUrl, 'schema')).rejects.toThrow(); + expect(execFileSyncMock).toHaveBeenCalledOnce(); + }); + + it('throws an error when the protocol is not calm:', async () => { + const calmHubUrl = 'https://not.calmhub.com/schemas/2025-03/meta/nonexistent.json'; + + await expect(calmHubDocumentLoader.loadMissingDocument(calmHubUrl, 'schema')).rejects.toThrow(); + expect(execFileSyncMock).not.toHaveBeenCalled(); + }); +}); \ No newline at end of file diff --git a/shared/src/document-loader/calmhub-custom-document-loader.ts b/shared/src/document-loader/calmhub-custom-document-loader.ts new file mode 100644 index 000000000..c4b9e8236 --- /dev/null +++ b/shared/src/document-loader/calmhub-custom-document-loader.ts @@ -0,0 +1,60 @@ +import { execFileSync } from 'child_process'; +import { SchemaDirectory } from '../schema-directory'; +import { CalmDocumentType, DocumentLoader, CALM_HUB_PROTO } from './document-loader'; +import { initLogger, Logger } from '../logger'; +import path from 'path'; +import { CALM_AUTH_PLUGIN_DIRECTORY } from '../consts'; + +export class CalmHubCustomDocumentLoader implements DocumentLoader { + private readonly logger: Logger; + private readonly baseURL: string; + private readonly wrapper: string; + + constructor(private calmHubUrl: string, calmHubWrapper: string, debug: boolean) { + this.baseURL = calmHubUrl; + this.wrapper = path.resolve(CALM_AUTH_PLUGIN_DIRECTORY, calmHubWrapper); + this.logger = initLogger(debug, 'calmhub-custom-document-loader'); + this.logger.info('Configuring CALMHub custom document loader with base URL: ' + calmHubUrl); + this.logger.info('Configuring CALMHub custom document loader with plugin: ' + this.wrapper); + } + + async initialise(_: SchemaDirectory): Promise { + return; + } + + async loadMissingDocument(documentId: string, _: CalmDocumentType): Promise { + const url = new URL(documentId); + const protocol = url.protocol; + if (protocol !== CALM_HUB_PROTO) { + throw new Error(`CalmhubCustomDocumentLoader only loads documents with protocol '${CALM_HUB_PROTO}'. (Requested: ${protocol})`); + } + const path = url.pathname; + + this.logger.info(`Loading CALM from ${this.calmHubUrl}${path}`); + + try { + const response = execFileSync(this.wrapper, ['--method', 'GET', this.baseURL + path], { + stdio: 'pipe', + shell: true, + encoding: 'utf-8', + timeout: 30000 // miliseconds + }); + this.logger.debug('Successfully loaded document from CALMHub with path ' + path); + this.logger.debug('' + response); + return JSON.parse(response); + } + catch (err) { + if (err.code) { + // Spawn failed + this.logger.error('CalmHub Wrapper spawn failed: ' + err.code); + } else { + // Wrapper executed but failed. + // Error contains stderr from child. + const { stderr } = err; + this.logger.error('CalmHub Wrapper error code & message: ' + err.status + ' / ' + stderr); + } + } + return document; + } + +} \ No newline at end of file diff --git a/shared/src/document-loader/document-loader.spec.ts b/shared/src/document-loader/document-loader.spec.ts index c5ce4171b..58783297f 100644 --- a/shared/src/document-loader/document-loader.spec.ts +++ b/shared/src/document-loader/document-loader.spec.ts @@ -10,6 +10,10 @@ const mocks = vi.hoisted(() => { calmHubDocLoader: vi.fn(() => ({ initialise: vi.fn(), loadMissingDocument: vi.fn() + })), + calmHubCustomDocLoader: vi.fn(() => ({ + initialise: vi.fn(), + loadMissingDocument: vi.fn() })) }; }); @@ -27,6 +31,12 @@ vi.mock('./calmhub-document-loader', () => { }; }); +vi.mock('./calmhub-custom-document-loader', () => { + return { + CalmHubCustomDocumentLoader: mocks.calmHubCustomDocLoader + }; +}); + describe('DocumentLoader', () => { beforeEach(() => { vi.clearAllMocks(); @@ -53,5 +63,29 @@ describe('DocumentLoader', () => { buildDocumentLoader(docLoaderOpts); expect(mocks.calmHubDocLoader).toHaveBeenCalledWith('https://example.com', false); + expect(mocks.calmHubCustomDocLoader).not.toHaveBeenCalled(); + }); + + it('should create a CalmHubCustomDocumentLoader when calmHubUrl and calmHubPlugin are defined in loader options', () => { + const docLoaderOpts: DocumentLoaderOptions = { + calmHubUrl: 'https://example.com', + calmHubPlugin: 'my-calmhub-plugin' + }; + + buildDocumentLoader(docLoaderOpts); + + expect(mocks.calmHubDocLoader).not.toHaveBeenCalled(); + expect(mocks.calmHubCustomDocLoader).toHaveBeenCalledWith('https://example.com', 'my-calmhub-plugin', false); + }); + + it('should not create a CalmHubDocumentLoader or CalmHubCustomDocumentLoader if calmHubUrl is not defined in loader options', () => { + const docLoaderOpts: DocumentLoaderOptions = { + calmHubPlugin: 'my-calmhub-plugin' + }; + + buildDocumentLoader(docLoaderOpts); + + expect(mocks.calmHubDocLoader).not.toHaveBeenCalled(); + expect(mocks.calmHubCustomDocLoader).not.toHaveBeenCalled(); }); }); \ No newline at end of file diff --git a/shared/src/document-loader/document-loader.ts b/shared/src/document-loader/document-loader.ts index 2f11f167f..d2c7fb701 100644 --- a/shared/src/document-loader/document-loader.ts +++ b/shared/src/document-loader/document-loader.ts @@ -1,6 +1,7 @@ import { CALM_META_SCHEMA_DIRECTORY } from '../consts'; import { SchemaDirectory } from '../schema-directory'; import { CalmHubDocumentLoader } from './calmhub-document-loader'; +import { CalmHubCustomDocumentLoader } from './calmhub-custom-document-loader'; import { FileSystemDocumentLoader } from './file-system-document-loader'; import { DirectUrlDocumentLoader } from './direct-url-document-loader'; import { MultiStrategyDocumentLoader } from './multi-strategy-document-loader'; @@ -16,6 +17,7 @@ export interface DocumentLoader { export type DocumentLoaderOptions = { calmHubUrl?: string; + calmHubPlugin?: string; schemaDirectoryPath?: string; debug?: boolean; }; @@ -25,7 +27,11 @@ export function buildDocumentLoader(docLoaderOpts: DocumentLoaderOptions): Docum const debug = docLoaderOpts.debug ?? false; if (docLoaderOpts.calmHubUrl) { - loaders.push(new CalmHubDocumentLoader(docLoaderOpts.calmHubUrl, debug)); + if (docLoaderOpts.calmHubPlugin) { + loaders.push(new CalmHubCustomDocumentLoader(docLoaderOpts.calmHubUrl, docLoaderOpts.calmHubPlugin, debug)); + } else { + loaders.push(new CalmHubDocumentLoader(docLoaderOpts.calmHubUrl, debug)); + } } // Always configure FileSystemDocumentLoader with CALM_META_SCHEMA_DIRECTORY