diff --git a/package-lock.json b/package-lock.json index cde5e9ed476..83a2233b43c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46849,6 +46849,7 @@ "@connectrpc/connect": "^1.4.0", "@connectrpc/connect-web": "^1.4.0", "@dagrejs/dagre": "^1.1.3", + "@exodus/schemasafe": "^1.3.0", "@floating-ui/dom": "^1.7.4", "@replit/codemirror-indentation-markers": "^6.5.3", "@storybook/addon-actions": "^7.6.17", diff --git a/runtime/drivers/azure/azure.go b/runtime/drivers/azure/azure.go index dbfd51ffd90..9bcd743bef8 100644 --- a/runtime/drivers/azure/azure.go +++ b/runtime/drivers/azure/azure.go @@ -39,12 +39,14 @@ var spec = drivers.Spec{ Secret: true, }, { - Key: "azure_storage_connection_string", - Type: drivers.StringPropertyType, - Secret: true, + Key: "azure_storage_connection_string", + Type: drivers.StringPropertyType, + DisplayName: "Azure Connection String", + Description: "Azure connection string for storage account", + Placeholder: "Paste your Azure connection string here", + Secret: true, }, }, - // Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas. SourceProperties: []*drivers.PropertySpec{ { Key: "path", diff --git a/runtime/drivers/duckdb/duckdb.go b/runtime/drivers/duckdb/duckdb.go index 12ee6fbaaf0..50c44c90b79 100644 --- a/runtime/drivers/duckdb/duckdb.go +++ b/runtime/drivers/duckdb/duckdb.go @@ -76,6 +76,14 @@ var spec = drivers.Spec{ Description: "Query to run on DuckDB.", Placeholder: "select * from table;", }, + { + Key: "create_secrets_from_connectors", + Type: drivers.StringPropertyType, + Required: false, + DisplayName: "Create secrets from connectors", + Description: "Comma-separated connector names to create temporary secrets before executing the SQL.", + NoPrompt: true, + }, }, ImplementsOLAP: true, } @@ -130,6 +138,14 @@ var motherduckSpec = drivers.Spec{ Description: "Query to extract data from MotherDuck.", Placeholder: "select * from table;", }, + { + Key: "create_secrets_from_connectors", + Type: drivers.StringPropertyType, + Required: false, + DisplayName: "Create secrets from connectors", + Description: "Comma-separated connector names to create temporary secrets before executing the SQL.", + NoPrompt: true, + }, }, ImplementsOLAP: true, } diff --git a/runtime/drivers/s3/s3.go b/runtime/drivers/s3/s3.go index 90e84dfd0ca..005e7ab4868 100644 --- a/runtime/drivers/s3/s3.go +++ b/runtime/drivers/s3/s3.go @@ -26,14 +26,22 @@ var spec = drivers.Spec{ DocsURL: "https://docs.rilldata.com/build/connectors/data-source/s3", ConfigProperties: []*drivers.PropertySpec{ { - Key: "aws_access_key_id", - Type: drivers.StringPropertyType, - Secret: true, + Key: "aws_access_key_id", + Type: drivers.StringPropertyType, + DisplayName: "AWS access key ID", + Description: "AWS access key ID for explicit credentials", + Placeholder: "Enter your AWS access key ID", + Secret: true, + Required: true, }, { - Key: "aws_secret_access_key", - Type: drivers.StringPropertyType, - Secret: true, + Key: "aws_secret_access_key", + Type: drivers.StringPropertyType, + DisplayName: "AWS secret access key", + Description: "AWS secret access key for explicit credentials", + Placeholder: "Enter your AWS secret access key", + Secret: true, + Required: true, }, { Key: "region", diff --git a/web-common/package.json b/web-common/package.json index 9acd75bfccb..9b89aa91a61 100644 --- a/web-common/package.json +++ b/web-common/package.json @@ -34,6 +34,7 @@ "@connectrpc/connect": "^1.4.0", "@connectrpc/connect-web": "^1.4.0", "@dagrejs/dagre": "^1.1.3", + "@exodus/schemasafe": "^1.3.0", "@floating-ui/dom": "^1.7.4", "@replit/codemirror-indentation-markers": "^6.5.3", "@storybook/addon-actions": "^7.6.17", diff --git a/web-common/src/features/entity-management/name-utils.ts b/web-common/src/features/entity-management/name-utils.ts index 01f9567a937..52a6d72dc99 100644 --- a/web-common/src/features/entity-management/name-utils.ts +++ b/web-common/src/features/entity-management/name-utils.ts @@ -21,8 +21,22 @@ export function getName(name: string, others: string[]): string { const set = new Set(others.map((other) => other.toLowerCase())); let result = name; + const incrementableSuffix = /(.+)_([0-9]+)$/; while (set.has(result.toLowerCase())) { + // Special-case for "s3": don't roll over to "s4", append suffix instead. + if (name.toLowerCase() === "s3") { + const match = incrementableSuffix.exec(result); + if (match) { + const base = match[1]; + const number = Number.parseInt(match[2], 10) + 1; + result = `${base}_${number}`; + continue; + } + result = `${name}_1`; + continue; + } + result = INCREMENT.exec(result)?.[1] ? result.replace(INCREMENT, (m) => (+m + 1).toString()) : `${result}_1`; diff --git a/web-common/src/features/sources/modal/AddClickHouseForm.svelte b/web-common/src/features/sources/modal/AddClickHouseForm.svelte index 3248cf7487d..5aa3a74d6ee 100644 --- a/web-common/src/features/sources/modal/AddClickHouseForm.svelte +++ b/web-common/src/features/sources/modal/AddClickHouseForm.svelte @@ -20,7 +20,8 @@ import type { ConnectorType } from "./types"; import { dsnSchema, getYupSchema } from "./yupSchemas"; import Checkbox from "@rilldata/web-common/components/forms/Checkbox.svelte"; - import { isEmpty, normalizeErrors } from "./utils"; + import { normalizeErrors } from "../../templates/error-utils"; + import { isEmpty } from "./utils"; import { CONNECTOR_TYPE_OPTIONS, CONNECTION_TAB_OPTIONS, diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index ee081dc8f34..bfbffa2efe8 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -3,28 +3,31 @@ import SubmissionError from "@rilldata/web-common/components/forms/SubmissionError.svelte"; import { queryClient } from "@rilldata/web-common/lib/svelte-query/globalQueryClient"; - import { type V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; + import { + type ConnectorDriverProperty, + type V1ConnectorDriver, + } from "@rilldata/web-common/runtime-client"; import type { ActionResult } from "@sveltejs/kit"; import type { SuperValidated } from "sveltekit-superforms"; import type { AddDataFormType, ConnectorType } from "./types"; import AddClickHouseForm from "./AddClickHouseForm.svelte"; + import ConnectorForm from "./ConnectorForm.svelte"; import NeedHelpText from "./NeedHelpText.svelte"; import Tabs from "@rilldata/web-common/components/forms/Tabs.svelte"; import { TabsContent } from "@rilldata/web-common/components/tabs"; - import { isEmpty } from "./utils"; + import { hasOnlyDsn, isEmpty } from "./utils"; import { CONNECTION_TAB_OPTIONS, type ClickHouseConnectorType, + FORM_HEIGHT_DEFAULT, + MULTI_STEP_CONNECTORS, } from "./constants"; - import { getInitialFormValuesFromProperties } from "../sourceUtils"; - import { connectorStepStore } from "./connectorStepStore"; import FormRenderer from "./FormRenderer.svelte"; import YamlPreview from "./YamlPreview.svelte"; - import GCSMultiStepForm from "./GCSMultiStepForm.svelte"; + import { AddDataFormManager } from "./AddDataFormManager"; - import { hasOnlyDsn } from "./utils"; import AddDataFormSection from "./AddDataFormSection.svelte"; export let connector: V1ConnectorDriver; @@ -33,9 +36,14 @@ export let onBack: () => void; export let onClose: () => void; + const isMultiStepConnector = + formType === "connector" && + MULTI_STEP_CONNECTORS.includes(connector.name ?? ""); + let saveAnyway = false; let showSaveAnyway = false; let connectionTab: ConnectorType = "parameters"; + let formHeight = FORM_HEIGHT_DEFAULT; // Wire manager-provided onUpdate after declaration below let handleOnUpdate: < @@ -49,78 +57,105 @@ result: Extract; }) => Promise = async (_event) => {}; - const formManager = new AddDataFormManager({ - connector, - formType, - onParamsUpdate: (e: any) => handleOnUpdate(e), - onDsnUpdate: (e: any) => handleOnUpdate(e), - }); - - const isMultiStepConnector = formManager.isMultiStepConnector; - const isSourceForm = formManager.isSourceForm; - const isConnectorForm = formManager.isConnectorForm; - const onlyDsn = hasOnlyDsn(connector, isConnectorForm); - $: stepState = $connectorStepStore; - $: stepProperties = - isMultiStepConnector && stepState.step === "source" - ? (connector.sourceProperties ?? []) - : properties; - $: if ( - isMultiStepConnector && - stepState.step === "source" && - stepState.connectorConfig - ) { - // Initialize form with source properties and default values - const sourceProperties = connector.sourceProperties ?? []; - const initialValues = getInitialFormValuesFromProperties(sourceProperties); - - // Merge with stored connector config - const combinedValues = { ...stepState.connectorConfig, ...initialValues }; - - paramsForm.update(() => combinedValues, { taint: false }); - } + let formManager: AddDataFormManager | null = null; - // Update form when (re)entering step 1: restore defaults for connector properties - $: if (isMultiStepConnector && stepState.step === "connector") { - paramsForm.update( - () => - getInitialFormValuesFromProperties(connector.configProperties ?? []), - { taint: false }, - ); + $: if (!isMultiStepConnector) { + formManager = new AddDataFormManager({ + connector, + formType, + onParamsUpdate: (e: any) => handleOnUpdate(e), + onDsnUpdate: (e: any) => handleOnUpdate(e), + getSelectedAuthMethod: () => undefined, + }); + formHeight = formManager.formHeight; } - // Form 1: Individual parameters - const paramsFormId = formManager.paramsFormId; - const properties = formManager.properties; - const filteredParamsProperties = formManager.filteredParamsProperties; - const { - form: paramsForm, - errors: paramsErrors, - enhance: paramsEnhance, - tainted: paramsTainted, - submit: paramsSubmit, - submitting: paramsSubmitting, - } = formManager.params; + let isSourceForm = formType === "source"; + let isConnectorForm = formType === "connector"; + let onlyDsn = false; + let activeAuthMethod: string | null = null; + let prevAuthMethod: string | null = null; + let multiStepSubmitDisabled = false; + let multiStepButtonLabel = ""; + let multiStepLoadingCopy = ""; + let primaryButtonLabel = ""; + let primaryLoadingCopy = ""; + let shouldShowSkipLink = false; + let multiStepYamlPreview = ""; + let multiStepYamlPreviewTitle = "Connector preview"; + let multiStepSubmitting = false; + let multiStepShowSaveAnyway = false; + let multiStepSaveAnywayLoading = false; + let multiStepSaveAnywayHandler: () => Promise = async () => {}; + let multiStepHandleBack: () => void = () => onBack(); + let multiStepHandleSkip: () => void = () => {}; + + // Form 1: Individual parameters (non-multi-step) + let paramsFormId = ""; + let properties: ConnectorDriverProperty[] = []; + let filteredParamsProperties: ConnectorDriverProperty[] = []; + let multiStepFormId = ""; + let paramsForm: any = null; + let paramsErrors: any = null; + let paramsEnhance: any = null; + let paramsTainted: any = null; + let paramsSubmit: any = null; + let paramsSubmitting: any = null; let paramsError: string | null = null; let paramsErrorDetails: string | undefined = undefined; - // Form 2: DSN - // SuperForms are not meant to have dynamic schemas, so we use a different form instance for the DSN form - const hasDsnFormOption = formManager.hasDsnFormOption; - const dsnFormId = formManager.dsnFormId; - const dsnProperties = formManager.dsnProperties; - const filteredDsnProperties = formManager.filteredDsnProperties; - const { - form: dsnForm, - errors: dsnErrors, - enhance: dsnEnhance, - tainted: dsnTainted, - submit: dsnSubmit, - submitting: dsnSubmitting, - } = formManager.dsn; + // Form 2: DSN (non-multi-step) + let hasDsnFormOption = false; + let dsnFormId = ""; + let dsnProperties: ConnectorDriverProperty[] = []; + let filteredDsnProperties: ConnectorDriverProperty[] = []; + let dsnForm: any = null; + let dsnErrors: any = null; + let dsnEnhance: any = null; + let dsnTainted: any = null; + let dsnSubmit: any = null; + let dsnSubmitting: any = null; let dsnError: string | null = null; let dsnErrorDetails: string | undefined = undefined; + $: if (formManager) { + paramsFormId = formManager.paramsFormId; + properties = formManager.properties; + filteredParamsProperties = formManager.filteredParamsProperties; + multiStepFormId = paramsFormId; + ({ + form: paramsForm, + errors: paramsErrors, + enhance: paramsEnhance, + tainted: paramsTainted, + submit: paramsSubmit, + submitting: paramsSubmitting, + } = formManager.params); + + hasDsnFormOption = formManager.hasDsnFormOption; + dsnFormId = formManager.dsnFormId; + dsnProperties = formManager.dsnProperties; + filteredDsnProperties = formManager.filteredDsnProperties; + ({ + form: dsnForm, + errors: dsnErrors, + enhance: dsnEnhance, + tainted: dsnTainted, + submit: dsnSubmit, + submitting: dsnSubmitting, + } = formManager.dsn); + + isSourceForm = formManager.isSourceForm; + isConnectorForm = formManager.isConnectorForm; + hasDsnFormOption = formManager.hasDsnFormOption; + onlyDsn = hasOnlyDsn(connector, isConnectorForm); + } else { + isSourceForm = formType === "source"; + isConnectorForm = formType === "connector"; + hasDsnFormOption = false; + onlyDsn = false; + } + let clickhouseError: string | null = null; let clickhouseErrorDetails: string | undefined = undefined; let clickhouseFormId: string = ""; @@ -132,6 +167,12 @@ let clickhouseShowSaveAnyway: boolean = false; $: isSubmitDisabled = (() => { + if (isMultiStepConnector) { + return multiStepSubmitDisabled; + } + + if (!formManager) return true; + if (onlyDsn || connectionTab === "dsn") { // DSN form: check required DSN properties for (const property of dsnProperties) { @@ -149,12 +190,7 @@ return false; } else { // Parameters form: check required properties - // Use stepProperties for multi-step connectors, otherwise use properties - const propertiesToCheck = isMultiStepConnector - ? stepProperties - : properties; - - for (const property of propertiesToCheck) { + for (const property of properties) { if (property.required) { const key = String(property.key); const value = $paramsForm[key]; @@ -168,9 +204,13 @@ } })(); - $: formId = formManager.getActiveFormId({ connectionTab, onlyDsn }); + $: formId = isMultiStepConnector + ? multiStepFormId + : (formManager?.getActiveFormId({ connectionTab, onlyDsn }) ?? ""); $: submitting = (() => { + if (isMultiStepConnector) return multiStepSubmitting; + if (!formManager) return false; if (onlyDsn || connectionTab === "dsn") { return $dsnSubmitting; } else { @@ -178,10 +218,71 @@ } })(); + $: primaryButtonLabel = isMultiStepConnector + ? multiStepButtonLabel + : formManager + ? formManager.getPrimaryButtonLabel({ + isConnectorForm, + step: isSourceForm ? "source" : "connector", + submitting, + clickhouseConnectorType, + clickhouseSubmitting, + selectedAuthMethod: activeAuthMethod ?? undefined, + }) + : ""; + + $: primaryLoadingCopy = (() => { + if (connector.name === "clickhouse") return "Connecting..."; + if (isMultiStepConnector) return multiStepLoadingCopy; + return activeAuthMethod === "public" + ? "Continuing..." + : "Testing connection..."; + })(); + + // Clear Save Anyway state whenever auth method changes (any direction). + $: if (activeAuthMethod !== prevAuthMethod) { + prevAuthMethod = activeAuthMethod; + showSaveAnyway = false; + saveAnyway = false; + } + $: isSubmitting = submitting; - // Reset errors when form is modified + $: ctaDisabled = + connector.name === "clickhouse" + ? clickhouseSubmitting || clickhouseIsSubmitDisabled + : isMultiStepConnector + ? multiStepSubmitting || multiStepSubmitDisabled + : submitting || isSubmitDisabled; + $: ctaLoading = + connector.name === "clickhouse" + ? clickhouseSubmitting + : isMultiStepConnector + ? multiStepSubmitting + : submitting; + $: ctaLoadingCopy = isMultiStepConnector + ? multiStepLoadingCopy + : primaryLoadingCopy; + $: ctaLabel = isMultiStepConnector + ? multiStepButtonLabel + : primaryButtonLabel; + $: ctaFormId = + connector.name === "clickhouse" + ? clickhouseFormId + : isMultiStepConnector + ? multiStepFormId + : formId; + + $: effectiveYaml = isMultiStepConnector ? multiStepYamlPreview : yamlPreview; + $: effectiveYamlTitle = isMultiStepConnector + ? multiStepYamlPreviewTitle + : isSourceForm + ? "Model preview" + : "Connector preview"; + + // Reset errors when form is modified (non-multi-step paths) $: (() => { + if (isMultiStepConnector || !formManager) return; if (onlyDsn || connectionTab === "dsn") { if ($dsnTainted) dsnError = null; } else { @@ -189,8 +290,9 @@ } })(); - // Clear errors when switching tabs + // Clear errors when switching tabs (non-multi-step paths) $: (() => { + if (isMultiStepConnector || !formManager) return; if (hasDsnFormOption) { if (connectionTab === "dsn") { paramsError = null; @@ -208,6 +310,14 @@ return; } + // Multi-step connectors delegate to the container handler + if (isMultiStepConnector) { + await multiStepSaveAnywayHandler(); + return; + } + + if (!formManager) return; + // For other connectors, use manager helper saveAnyway = true; const values = @@ -251,49 +361,65 @@ } } - $: yamlPreview = formManager.computeYamlPreview({ - connectionTab, - onlyDsn, - filteredParamsProperties, - filteredDsnProperties, - stepState, - isMultiStepConnector, - isConnectorForm, - paramsFormValues: $paramsForm, - dsnFormValues: $dsnForm, - clickhouseConnectorType, - clickhouseParamsValues: $clickhouseParamsForm, - clickhouseDsnValues: $clickhouseDsnForm, - }); + $: yamlPreview = isMultiStepConnector + ? multiStepYamlPreview + : formManager + ? formManager.computeYamlPreview({ + connectionTab, + onlyDsn, + filteredParamsProperties, + filteredDsnProperties, + stepState: undefined, + isMultiStepConnector, + isConnectorForm, + paramsFormValues: $paramsForm, + dsnFormValues: $dsnForm, + clickhouseConnectorType, + clickhouseParamsValues: $clickhouseParamsForm, + clickhouseDsnValues: $clickhouseDsnForm, + }) + : ""; $: isClickhouse = connector.name === "clickhouse"; $: shouldShowSaveAnywayButton = - isConnectorForm && (showSaveAnyway || clickhouseShowSaveAnyway); - $: saveAnywayLoading = isClickhouse - ? clickhouseSubmitting && saveAnyway - : submitting && saveAnyway; - - handleOnUpdate = formManager.makeOnUpdate({ - onClose, - queryClient, - getConnectionTab: () => connectionTab, - setParamsError: (message: string | null, details?: string) => { - paramsError = message; - paramsErrorDetails = details; - }, - setDsnError: (message: string | null, details?: string) => { - dsnError = message; - dsnErrorDetails = details; - }, - setShowSaveAnyway: (value: boolean) => { - showSaveAnyway = value; - }, - }); + isConnectorForm && + (clickhouseShowSaveAnyway || + (isMultiStepConnector ? multiStepShowSaveAnyway : showSaveAnyway)); + $: saveAnywayLoading = isMultiStepConnector + ? multiStepSaveAnywayLoading + : isClickhouse + ? clickhouseSubmitting && saveAnyway + : submitting && saveAnyway; + + if (formManager) { + const fm = formManager as AddDataFormManager; + handleOnUpdate = + fm.makeOnUpdate({ + onClose, + queryClient, + getConnectionTab: () => connectionTab, + getSelectedAuthMethod: () => activeAuthMethod || undefined, + setParamsError: (message: string | null, details?: string) => { + paramsError = message; + paramsErrorDetails = details; + }, + setDsnError: (message: string | null, details?: string) => { + dsnError = message; + dsnErrorDetails = details; + }, + setShowSaveAnyway: (value: boolean) => { + showSaveAnyway = value; + }, + }) ?? (async () => {}); + } else { + handleOnUpdate = async () => {}; + } async function handleFileUpload(file: File): Promise { - return formManager.handleFileUpload(file); + return formManager ? formManager.handleFileUpload(file) : ""; } function onStringInputChange(event: Event) { + if (!formManager) return; formManager.onStringInputChange( event, $paramsTainted as Record | null | undefined, @@ -306,9 +432,7 @@
-
+
{#if connector.name === "clickhouse"} + {:else if isMultiStepConnector} + {#key connector.name} + + {/key} {:else if hasDsnFormOption} {:else if isConnectorForm && connector.configProperties?.some((property) => property.key === "dsn")} - - {:else if isMultiStepConnector} - {#if stepState.step === "connector"} - - - - - {:else} - - - - - {/if} {:else} - + isMultiStepConnector + ? multiStepHandleBack() + : formManager?.handleBack(onBack)} + type="secondary">Back
@@ -448,33 +566,15 @@ {/if} - {#if isMultiStepConnector && stepState.step === "connector"} - - {/if} -
@@ -482,31 +582,39 @@
- {#if dsnError || paramsError || clickhouseError} - - {/if} - - +
+ {#if dsnError || paramsError || clickhouseError} + + {/if} + + + + {#if shouldShowSkipLink} +
+ Already connected? +
+ {/if} +
diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index e2d0f9c4c10..7bb829a05cc 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -25,8 +25,11 @@ import { } from "./constants"; import { connectorStepStore, + resetConnectorStep, setConnectorConfig, + setAuthMethod, setStep, + type ConnectorStepState, } from "./connectorStepStore"; import { get } from "svelte/store"; import { compileConnectorYAML } from "../../connectors/code-utils"; @@ -35,16 +38,18 @@ import type { ConnectorDriverProperty } from "@rilldata/web-common/runtime-clien import type { ClickHouseConnectorType } from "./constants"; import { applyClickHouseCloudRequirements } from "./utils"; import type { ActionResult } from "@sveltejs/kit"; +import { getConnectorSchema } from "./connector-schemas"; +import { findRadioEnumKey } from "../../templates/schema-utils"; // Minimal onUpdate event type carrying Superforms's validated form type SuperFormUpdateEvent = { form: SuperValidated, any, Record>; }; -// Shape of the step store for multi-step connectors -type ConnectorStepState = { - step: "connector" | "source"; - connectorConfig: Record | null; +const BUTTON_LABELS = { + public: { idle: "Continue", submitting: "Continuing..." }, + connector: { idle: "Test and Connect", submitting: "Testing connection..." }, + source: { idle: "Import Data", submitting: "Importing data..." }, }; export class AddDataFormManager { @@ -63,21 +68,41 @@ export class AddDataFormManager { dsn: ReturnType; private connector: V1ConnectorDriver; private formType: AddDataFormType; + private initialParamsValues: Record; + private initialDsnValues: Record; // Centralized error normalization for this manager private normalizeError(e: unknown): { message: string; details?: string } { return normalizeConnectorError(this.connector.name ?? "", e); } + private getSelectedAuthMethod?: () => string | undefined; + private resetConnectorForms() { + if (this.params?.form) { + (this.params.form as any).update(() => ({}), { taint: false } as any); + } + if (this.dsn?.form) { + (this.dsn.form as any).update(() => ({}), { taint: false } as any); + } + } + constructor(args: { connector: V1ConnectorDriver; formType: AddDataFormType; onParamsUpdate: (event: SuperFormUpdateEvent) => void; onDsnUpdate: (event: SuperFormUpdateEvent) => void; + getSelectedAuthMethod?: () => string | undefined; }) { - const { connector, formType, onParamsUpdate, onDsnUpdate } = args; + const { + connector, + formType, + onParamsUpdate, + onDsnUpdate, + getSelectedAuthMethod, + } = args; this.connector = connector; this.formType = formType; + this.getSelectedAuthMethod = getSelectedAuthMethod; // Layout height this.formHeight = TALL_FORM_CONNECTORS.has(connector.name ?? "") @@ -125,15 +150,19 @@ export class AddDataFormManager { ); // Superforms: params - const paramsSchemaDef = getValidationSchemaForConnector( + const paramsAdapter = getValidationSchemaForConnector( connector.name as string, + formType, + { + isMultiStepConnector: this.isMultiStepConnector, + }, ); - const paramsAdapter = yup(paramsSchemaDef); - type ParamsOut = YupInfer; - type ParamsIn = YupInferIn; + type ParamsOut = Record; + type ParamsIn = Record; const initialFormValues = getInitialFormValuesFromProperties( this.properties, ); + this.initialParamsValues = initialFormValues; const paramsDefaults = defaults( initialFormValues as Partial, paramsAdapter, @@ -143,17 +172,21 @@ export class AddDataFormManager { validators: paramsAdapter, onUpdate: onParamsUpdate, resetForm: false, + validationMethod: "onsubmit", }); // Superforms: dsn const dsnAdapter = yup(dsnSchema); type DsnOut = YupInfer; type DsnIn = YupInferIn; - this.dsn = superForm(defaults(dsnAdapter), { + const initialDsnValues = defaults(dsnAdapter); + this.initialDsnValues = initialDsnValues; + this.dsn = superForm(initialDsnValues, { SPA: true, validators: dsnAdapter, onUpdate: onDsnUpdate, resetForm: false, + validationMethod: "onsubmit", }); } @@ -169,6 +202,40 @@ export class AddDataFormManager { return MULTI_STEP_CONNECTORS.includes(this.connector.name ?? ""); } + /** + * Determines whether the "Save Anyway" button should be shown for the current submission. + */ + private shouldShowSaveAnywayButton(args: { + isConnectorForm: boolean; + event?: + | { + result?: Extract; + } + | undefined; + stepState: ConnectorStepState | undefined; + selectedAuthMethod?: string; + }): boolean { + const { isConnectorForm, event, stepState, selectedAuthMethod } = args; + + // Only show for connector forms (not sources) + if (!isConnectorForm) return false; + + // ClickHouse has its own error handling + if (this.connector.name === "clickhouse") return false; + + // Need a submission result to show the button + if (!event?.result) return false; + + // Multi-step connectors: don't show on source step (final step) + if (stepState?.step === "source") return false; + + // Public auth bypasses connection test, so no "Save Anyway" needed + if (stepState?.step === "connector" && selectedAuthMethod === "public") + return false; + + return true; + } + getActiveFormId(args: { connectionTab: "parameters" | "dsn"; onlyDsn: boolean; @@ -182,13 +249,19 @@ export class AddDataFormManager { handleSkip(): void { const stepState = get(connectorStepStore) as ConnectorStepState; if (!this.isMultiStepConnector || stepState.step !== "connector") return; - setConnectorConfig(get(this.params.form) as Record); + setConnectorConfig({}); + setAuthMethod(null); + this.resetConnectorForms(); setStep("source"); } handleBack(onBack: () => void): void { const stepState = get(connectorStepStore) as ConnectorStepState; if (this.isMultiStepConnector && stepState.step === "source") { + // Clear any connector form state when navigating back from the model step + setConnectorConfig(null); + setAuthMethod(null); + this.resetConnectorForms(); setStep("connector"); } else { onBack(); @@ -201,6 +274,7 @@ export class AddDataFormManager { submitting: boolean; clickhouseConnectorType?: ClickHouseConnectorType; clickhouseSubmitting?: boolean; + selectedAuthMethod?: string; }): string { const { isConnectorForm, @@ -208,6 +282,7 @@ export class AddDataFormManager { submitting, clickhouseConnectorType, clickhouseSubmitting, + selectedAuthMethod, } = args; const isClickhouse = this.connector.name === "clickhouse"; @@ -222,12 +297,23 @@ export class AddDataFormManager { if (isConnectorForm) { if (this.isMultiStepConnector && step === "connector") { - return submitting ? "Testing connection..." : "Test and Connect"; + if (selectedAuthMethod === "public") { + return submitting + ? BUTTON_LABELS.public.submitting + : BUTTON_LABELS.public.idle; + } + return submitting + ? BUTTON_LABELS.connector.submitting + : BUTTON_LABELS.connector.idle; } if (this.isMultiStepConnector && step === "source") { - return submitting ? "Creating model..." : "Test and Add data"; + return submitting + ? BUTTON_LABELS.source.submitting + : BUTTON_LABELS.source.idle; } - return submitting ? "Testing connection..." : "Test and Connect"; + return submitting + ? BUTTON_LABELS.connector.submitting + : BUTTON_LABELS.connector.idle; } return "Test and Add data"; @@ -237,6 +323,7 @@ export class AddDataFormManager { onClose: () => void; queryClient: any; getConnectionTab: () => "parameters" | "dsn"; + getSelectedAuthMethod?: () => string | undefined; setParamsError: (message: string | null, details?: string) => void; setDsnError: (message: string | null, details?: string) => void; setShowSaveAnyway?: (value: boolean) => void; @@ -245,6 +332,7 @@ export class AddDataFormManager { onClose, queryClient, getConnectionTab, + getSelectedAuthMethod, setParamsError, setDsnError, setShowSaveAnyway, @@ -262,33 +350,92 @@ export class AddDataFormManager { Record >; result?: Extract; + cancel?: () => void; }) => { - // For non-ClickHouse connectors, expose Save Anyway when a submission starts + const values = event.form.data; + const schema = getConnectorSchema(this.connector.name ?? ""); + const authKey = schema ? findRadioEnumKey(schema) : null; + const selectedAuthMethod = + (authKey && values && values[authKey] != null + ? String(values[authKey]) + : undefined) || + getSelectedAuthMethod?.() || + ""; + const stepState = get(connectorStepStore) as ConnectorStepState; + + // Fast-path: public auth skips validation/test and goes straight to source step. if ( - isConnectorForm && - connector.name !== "clickhouse" && - typeof setShowSaveAnyway === "function" && - event?.result + isMultiStepConnector && + stepState.step === "connector" && + selectedAuthMethod === "public" ) { - setShowSaveAnyway(true); + setConnectorConfig({}); + setAuthMethod(null); + setStep("source"); + return; } - if (!event.form.valid) return; + if (isMultiStepConnector && stepState.step === "source") { + const sourceValidator = getValidationSchemaForConnector( + connector.name as string, + "source", + { isMultiStepConnector: true }, + ); + const result = await sourceValidator.validate(values); + if (!result.success) { + const fieldErrors: Record = {}; + for (const issue of result.issues ?? []) { + const key = + issue.path?.[0] != null ? String(issue.path[0]) : "_errors"; + if (!fieldErrors[key]) fieldErrors[key] = []; + fieldErrors[key].push(issue.message); + } + (this.params.errors as any).set(fieldErrors); + event.cancel?.(); + return; + } + (this.params.errors as any).set({}); + } else if (!event.form.valid) { + return; + } - const values = event.form.data; + if ( + typeof setShowSaveAnyway === "function" && + this.shouldShowSaveAnywayButton({ + isConnectorForm, + event, + stepState, + selectedAuthMethod, + }) + ) { + setShowSaveAnyway(true); + } try { - const stepState = get(connectorStepStore) as ConnectorStepState; if (isMultiStepConnector && stepState.step === "source") { await submitAddSourceForm(queryClient, connector, values); + resetConnectorStep(); + this.resetConnectorForms(); onClose(); } else if (isMultiStepConnector && stepState.step === "connector") { + // For public auth, skip Test & Connect and go straight to the next step. + if (selectedAuthMethod === "public") { + setConnectorConfig({}); + setAuthMethod(null); + this.resetConnectorForms(); + setStep("source"); + return; + } await submitAddConnectorForm(queryClient, connector, values, false); - setConnectorConfig(values); + setConnectorConfig({}); + setAuthMethod(null); + this.resetConnectorForms(); setStep("source"); return; } else if (this.formType === "source") { await submitAddSourceForm(queryClient, connector, values); + resetConnectorStep(); + this.resetConnectorForms(); onClose(); } else { await submitAddConnectorForm(queryClient, connector, values, false); @@ -392,16 +539,22 @@ export class AddDataFormManager { clickhouseDsnValues, } = ctx; + const connectorPropertiesForPreview = + isMultiStepConnector && stepState?.step === "connector" + ? (connector.configProperties ?? []) + : filteredParamsProperties; + const getConnectorYamlPreview = (values: Record) => { + const orderedProperties = + onlyDsn || connectionTab === "dsn" + ? filteredDsnProperties + : connectorPropertiesForPreview; return compileConnectorYAML(connector, values, { fieldFilter: (property) => { if (onlyDsn || connectionTab === "dsn") return true; return !property.noPrompt; }, - orderedProperties: - onlyDsn || connectionTab === "dsn" - ? filteredDsnProperties - : filteredParamsProperties, + orderedProperties, }); }; diff --git a/web-common/src/features/sources/modal/AddDataModal.svelte b/web-common/src/features/sources/modal/AddDataModal.svelte index 42c8289d1b1..e2dd69ade1e 100644 --- a/web-common/src/features/sources/modal/AddDataModal.svelte +++ b/web-common/src/features/sources/modal/AddDataModal.svelte @@ -71,6 +71,9 @@ }); function goToConnectorForm(connector: V1ConnectorDriver) { + // Reset multi-step state (auth selection, connector config) when switching connectors. + resetConnectorStep(); + const state = { step: 2, selectedConnector: connector, @@ -121,7 +124,7 @@ // FIXME: excluding salesforce until we implement the table discovery APIs $: isConnectorType = - selectedConnector?.name === "gcs" || + selectedConnector?.implementsObjectStore || selectedConnector?.implementsOlap || selectedConnector?.implementsSqlStore || (selectedConnector?.implementsWarehouse && diff --git a/web-common/src/features/sources/modal/ConnectorForm.svelte b/web-common/src/features/sources/modal/ConnectorForm.svelte new file mode 100644 index 00000000000..513b389277d --- /dev/null +++ b/web-common/src/features/sources/modal/ConnectorForm.svelte @@ -0,0 +1,277 @@ + + + diff --git a/web-common/src/features/sources/modal/FormRenderer.svelte b/web-common/src/features/sources/modal/FormRenderer.svelte index a114345627f..dc33afc5fff 100644 --- a/web-common/src/features/sources/modal/FormRenderer.svelte +++ b/web-common/src/features/sources/modal/FormRenderer.svelte @@ -7,7 +7,7 @@ ConnectorDriverPropertyType, type ConnectorDriverProperty, } from "@rilldata/web-common/runtime-client"; - import { normalizeErrors } from "./utils"; + import { normalizeErrors } from "../../templates/error-utils"; export let properties: Array = []; export let form: any; // expect a store from parent diff --git a/web-common/src/features/sources/modal/FormValidation.test.ts b/web-common/src/features/sources/modal/FormValidation.test.ts new file mode 100644 index 00000000000..9df8111c168 --- /dev/null +++ b/web-common/src/features/sources/modal/FormValidation.test.ts @@ -0,0 +1,74 @@ +import { describe, it, expect } from "vitest"; + +import { getValidationSchemaForConnector } from "./FormValidation"; + +describe("getValidationSchemaForConnector (multi-step auth)", () => { + it("enforces required fields for access key auth", async () => { + const schema = getValidationSchemaForConnector("s3", "connector", { + isMultiStepConnector: true, + }); + + const result = await schema.validate({}); + expect(result.success).toBe(false); + if (result.success) throw new Error("expected validation to fail"); + expect(result.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ path: ["aws_access_key_id"] }), + expect.objectContaining({ path: ["aws_secret_access_key"] }), + ]), + ); + }); + + it("allows public auth without credentials", async () => { + const schema = getValidationSchemaForConnector("s3", "connector", { + isMultiStepConnector: true, + }); + + const result = await schema.validate({ auth_method: "public" }); + expect(result.success).toBe(true); + }); + + it("requires source fields from JSON schema for multi-step connectors", async () => { + const schema = getValidationSchemaForConnector("s3", "source", { + isMultiStepConnector: true, + }); + + const result = await schema.validate({}); + expect(result.success).toBe(false); + if (result.success) throw new Error("expected validation to fail"); + expect(result.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ path: ["path"] }), + expect.objectContaining({ path: ["name"] }), + ]), + ); + }); + + it("rejects invalid s3 path on source step", async () => { + const schema = getValidationSchemaForConnector("s3", "source", { + isMultiStepConnector: true, + }); + + const result = await schema.validate({ + path: "s3:/bucket", + name: "valid_name", + }); + expect(result.success).toBe(false); + if (result.success) throw new Error("expected validation to fail"); + expect(result.issues).toEqual( + expect.arrayContaining([expect.objectContaining({ path: ["path"] })]), + ); + }); + + it("accepts valid s3 path on source step", async () => { + const schema = getValidationSchemaForConnector("s3", "source", { + isMultiStepConnector: true, + }); + + const result = await schema.validate({ + path: "s3://bucket/prefix", + name: "valid_name", + }); + expect(result.success).toBe(true); + }); +}); diff --git a/web-common/src/features/sources/modal/FormValidation.ts b/web-common/src/features/sources/modal/FormValidation.ts index 92c0f87c338..568d6862f7d 100644 --- a/web-common/src/features/sources/modal/FormValidation.ts +++ b/web-common/src/features/sources/modal/FormValidation.ts @@ -1,7 +1,28 @@ +import type { ValidationAdapter } from "sveltekit-superforms/adapters"; +import { yup as yupAdapter } from "sveltekit-superforms/adapters"; + +import { createSchemasafeValidator } from "./jsonSchemaValidator"; +import { getConnectorSchema } from "./connector-schemas"; import { dsnSchema, getYupSchema } from "./yupSchemas"; +import type { AddDataFormType } from "./types"; export { dsnSchema }; -export function getValidationSchemaForConnector(name: string) { - return getYupSchema[name as keyof typeof getYupSchema]; +export function getValidationSchemaForConnector( + name: string, + formType: AddDataFormType, + opts?: { + isMultiStepConnector?: boolean; + }, +): ValidationAdapter> { + const { isMultiStepConnector } = opts || {}; + const jsonSchema = getConnectorSchema(name); + + if (isMultiStepConnector && jsonSchema) { + const step = formType === "source" ? "source" : "connector"; + return createSchemasafeValidator(jsonSchema, step); + } + + const fallbackYupSchema = getYupSchema[name as keyof typeof getYupSchema]; + return yupAdapter(fallbackYupSchema); } diff --git a/web-common/src/features/sources/modal/GCSMultiStepForm.svelte b/web-common/src/features/sources/modal/GCSMultiStepForm.svelte deleted file mode 100644 index e007489218a..00000000000 --- a/web-common/src/features/sources/modal/GCSMultiStepForm.svelte +++ /dev/null @@ -1,129 +0,0 @@ - - - -
-
-
Authentication method
- - - {#if option.value === "credentials"} - - {:else if option.value === "hmac"} -
- - -
- {/if} - - -
- - - {#each filteredParamsProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} - {#if propertyKey !== "path" && propertyKey !== "google_application_credentials" && propertyKey !== "key_id" && propertyKey !== "secret"} -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {/if} -
- {/if} - {/each} -
diff --git a/web-common/src/features/sources/modal/connector-schemas.ts b/web-common/src/features/sources/modal/connector-schemas.ts new file mode 100644 index 00000000000..0b7cff5363b --- /dev/null +++ b/web-common/src/features/sources/modal/connector-schemas.ts @@ -0,0 +1,19 @@ +import type { MultiStepFormSchema } from "../../templates/schemas/types"; +import { azureSchema } from "../../templates/schemas/azure"; +import { gcsSchema } from "../../templates/schemas/gcs"; +import { s3Schema } from "../../templates/schemas/s3"; + +export const multiStepFormSchemas: Record = { + s3: s3Schema, + gcs: gcsSchema, + azure: azureSchema, +}; + +export function getConnectorSchema( + connectorName: string, +): MultiStepFormSchema | null { + const schema = + multiStepFormSchemas[connectorName as keyof typeof multiStepFormSchemas]; + if (!schema?.properties) return null; + return schema; +} diff --git a/web-common/src/features/sources/modal/connectorStepStore.ts b/web-common/src/features/sources/modal/connectorStepStore.ts index 9ce2451f126..8a20232e9be 100644 --- a/web-common/src/features/sources/modal/connectorStepStore.ts +++ b/web-common/src/features/sources/modal/connectorStepStore.ts @@ -2,25 +2,37 @@ import { writable } from "svelte/store"; export type ConnectorStep = "connector" | "source"; -export const connectorStepStore = writable<{ +export type ConnectorStepState = { step: ConnectorStep; connectorConfig: Record | null; -}>({ + selectedAuthMethod: string | null; +}; + +export const connectorStepStore = writable({ step: "connector", connectorConfig: null, + selectedAuthMethod: null, }); export function setStep(step: ConnectorStep) { connectorStepStore.update((state) => ({ ...state, step })); } -export function setConnectorConfig(config: Record) { +export function setConnectorConfig(config: Record | null) { connectorStepStore.update((state) => ({ ...state, connectorConfig: config })); } +export function setAuthMethod(method: string | null) { + connectorStepStore.update((state) => ({ + ...state, + selectedAuthMethod: method, + })); +} + export function resetConnectorStep() { connectorStepStore.set({ step: "connector", connectorConfig: null, + selectedAuthMethod: null, }); } diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 4bffc52825d..8e02f8a7b44 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -17,7 +17,7 @@ export const CONNECTION_TAB_OPTIONS: { value: string; label: string }[] = [ { value: "dsn", label: "Enter connection string" }, ]; -export type GCSAuthMethod = "credentials" | "hmac"; +export type GCSAuthMethod = "public" | "credentials" | "hmac"; export const GCS_AUTH_OPTIONS: { value: GCSAuthMethod; @@ -37,6 +37,65 @@ export const GCS_AUTH_OPTIONS: { description: "Use HMAC access key and secret for S3-compatible authentication.", }, + { + value: "public", + label: "Public", + description: "Access publicly readable buckets without credentials.", + }, +]; + +export type S3AuthMethod = "access_keys" | "public"; + +export const S3_AUTH_OPTIONS: { + value: S3AuthMethod; + label: string; + description: string; + hint?: string; +}[] = [ + { + value: "access_keys", + label: "Access keys", + description: "Use AWS access key ID and secret access key.", + }, + { + value: "public", + label: "Public", + description: "Access publicly readable buckets without credentials.", + }, +]; + +export type AzureAuthMethod = + | "account_key" + | "sas_token" + | "connection_string" + | "public"; + +export const AZURE_AUTH_OPTIONS: { + value: AzureAuthMethod; + label: string; + description: string; + hint?: string; +}[] = [ + { + value: "connection_string", + label: "Connection String", + description: "Alternative for cloud deployment", + }, + { + value: "account_key", + label: "Storage Account Key", + description: "Recommended for cloud deployment", + }, + { + value: "sas_token", + label: "Shared Access Signature (SAS) Token", + description: "Most secure, fine-grained control", + }, + { + value: "public", + label: "Public", + description: "Access publicly readable blobs without credentials.", + }, ]; // pre-defined order for sources @@ -67,7 +126,7 @@ export const OLAP_ENGINES = [ export const ALL_CONNECTORS = [...SOURCES, ...OLAP_ENGINES]; // Connectors that support multi-step forms (connector -> source) -export const MULTI_STEP_CONNECTORS = ["gcs"]; +export const MULTI_STEP_CONNECTORS = ["gcs", "s3", "azure"]; export const FORM_HEIGHT_TALL = "max-h-[38.5rem] min-h-[38.5rem]"; export const FORM_HEIGHT_DEFAULT = "max-h-[34.5rem] min-h-[34.5rem]"; diff --git a/web-common/src/features/sources/modal/jsonSchemaValidator.ts b/web-common/src/features/sources/modal/jsonSchemaValidator.ts new file mode 100644 index 00000000000..1804f0713b4 --- /dev/null +++ b/web-common/src/features/sources/modal/jsonSchemaValidator.ts @@ -0,0 +1,193 @@ +import type { ValidatorOptions, ValidationError } from "@exodus/schemasafe"; +import { validator as compileValidator } from "@exodus/schemasafe"; +import { schemasafe } from "sveltekit-superforms/adapters"; +import type { ValidationAdapter } from "sveltekit-superforms/adapters"; + +import { getFieldLabel, isStepMatch } from "../../templates/schema-utils"; +import type { + JSONSchemaConditional, + JSONSchemaConstraint, + MultiStepFormSchema, +} from "../../templates/schemas/types"; + +const DEFAULT_SCHEMASAFE_OPTIONS: ValidatorOptions = { + includeErrors: true, + allErrors: true, + allowUnusedKeywords: true, + formats: { + uri: (value: string) => { + if (typeof value !== "string") return false; + try { + // Allow custom schemes such as s3:// or gs:// + new URL(value); + return true; + } catch { + return false; + } + }, + // We treat file inputs as strings; superforms handles the upload. + file: () => true, + }, +}; + +type Step = "connector" | "source" | string | undefined; + +export function buildStepSchema( + schema: MultiStepFormSchema, + step: Step, +): MultiStepFormSchema { + const properties = Object.entries(schema.properties ?? {}).reduce< + NonNullable + >((acc, [key, prop]) => { + if (!isStepMatch(schema, key, step)) return acc; + acc[key] = prop; + return acc; + }, {}); + + const required = (schema.required ?? []).filter((key) => + isStepMatch(schema, key, step), + ); + + const filteredAllOf = (schema.allOf ?? []) + .map((conditional) => filterConditional(conditional, schema, step)) + .filter((conditional): conditional is JSONSchemaConditional => + Boolean( + conditional && + ((conditional.then?.required?.length ?? 0) > 0 || + (conditional.else?.required?.length ?? 0) > 0), + ), + ); + + return { + $schema: schema.$schema, + type: "object", + properties, + ...(required.length ? { required } : {}), + ...(filteredAllOf.length ? { allOf: filteredAllOf } : {}), + }; +} + +export function createSchemasafeValidator( + schema: MultiStepFormSchema, + step: Step, + opts?: { config?: ValidatorOptions }, +): ValidationAdapter> { + const stepSchema = buildStepSchema(schema, step); + const validator = compileValidator(stepSchema, { + ...DEFAULT_SCHEMASAFE_OPTIONS, + ...opts?.config, + }); + + const baseAdapter = schemasafe(stepSchema, { + config: { + ...DEFAULT_SCHEMASAFE_OPTIONS, + ...opts?.config, + }, + }); + + return { + ...baseAdapter, + async validate(data: Record = {}) { + const pruned = pruneEmptyFields(data); + const isValid = validator(pruned as any); + if (isValid) { + return { data: pruned, success: true }; + } + + const issues = (validator.errors ?? []).map((error) => + toIssue(error, schema), + ); + return { success: false, issues }; + }, + }; +} + +function pruneEmptyFields( + values: Record, +): Record { + const result: Record = {}; + for (const [key, value] of Object.entries(values ?? {})) { + if (value === "" || value === null || value === undefined) continue; + result[key] = value; + } + return result; +} + +function filterConditional( + conditional: JSONSchemaConditional, + schema: MultiStepFormSchema, + step: Step, +): JSONSchemaConditional | null { + const thenRequired = filterRequired(conditional.then, schema, step); + const elseRequired = filterRequired(conditional.else, schema, step); + + if (!thenRequired.length && !elseRequired.length) return null; + + return { + if: conditional.if, + then: thenRequired.length ? { required: thenRequired } : undefined, + else: elseRequired.length ? { required: elseRequired } : undefined, + }; +} + +function filterRequired( + constraint: JSONSchemaConstraint | undefined, + schema: MultiStepFormSchema, + step: Step, +): string[] { + return (constraint?.required ?? []).filter((key) => + isStepMatch(schema, key, step), + ); +} + +function toIssue(error: ValidationError, schema: MultiStepFormSchema) { + const pathSegments = parseInstanceLocation(error.instanceLocation); + const key = pathSegments[0]; + return { + path: pathSegments, + message: buildMessage(schema, key, error), + }; +} + +function buildMessage( + schema: MultiStepFormSchema, + key: string | undefined, + error: ValidationError, +): string { + if (!key) return "Invalid value"; + const prop = schema.properties?.[key] as any; + const label = getFieldLabel(schema, key); + const keyword = parseKeyword(error.keywordLocation); + + if (keyword === "required") return `${label} is required`; + + if (keyword === "pattern") { + const custom = prop?.errorMessage?.pattern as string | undefined; + return custom || `${label} is invalid`; + } + + if (keyword === "format") { + const custom = prop?.errorMessage?.format as string | undefined; + const format = prop?.format as string | undefined; + if (custom) return custom; + if (format) return `${label} must be a valid ${format}`; + return `${label} is invalid`; + } + + if (keyword === "type" && prop?.type) { + return `${label} must be a ${prop.type}`; + } + + return `${label} is invalid`; +} + +function parseInstanceLocation(location: string): string[] { + if (!location || location === "#") return []; + return location.replace(/^#\//, "").split("/").filter(Boolean); +} + +function parseKeyword(location: string): string { + if (!location) return ""; + const parts = location.split("/"); + return parts[parts.length - 1] || ""; +} diff --git a/web-common/src/features/sources/modal/submitAddDataForm.ts b/web-common/src/features/sources/modal/submitAddDataForm.ts index c6ec540ad6e..d9bdbd6f332 100644 --- a/web-common/src/features/sources/modal/submitAddDataForm.ts +++ b/web-common/src/features/sources/modal/submitAddDataForm.ts @@ -154,88 +154,6 @@ async function getOriginalEnvBlob( } } -export async function submitAddSourceForm( - queryClient: QueryClient, - connector: V1ConnectorDriver, - formValues: AddDataFormValues, -): Promise { - const instanceId = get(runtime).instanceId; - await beforeSubmitForm(instanceId, connector); - - const newSourceName = formValues.name as string; - - const [rewrittenConnector, rewrittenFormValues] = prepareSourceFormData( - connector, - formValues, - ); - - // Make a new .yaml file - const newSourceFilePath = getFileAPIPathFromNameAndType( - newSourceName, - EntityType.Table, - ); - await runtimeServicePutFile(instanceId, { - path: newSourceFilePath, - blob: compileSourceYAML(rewrittenConnector, rewrittenFormValues), - create: true, - createOnly: false, - }); - - const originalEnvBlob = await getOriginalEnvBlob(queryClient, instanceId); - - // Create or update the `.env` file - const newEnvBlob = await updateDotEnvWithSecrets( - queryClient, - rewrittenConnector, - rewrittenFormValues, - "source", - ); - - // Make sure the file has reconciled before testing the connection - await runtimeServicePutFileAndWaitForReconciliation(instanceId, { - path: ".env", - blob: newEnvBlob, - create: true, - createOnly: false, - }); - - // Wait for source resource-level reconciliation - // This must happen after .env reconciliation since sources depend on secrets - try { - await waitForResourceReconciliation( - instanceId, - newSourceName, - ResourceKind.Model, - ); - } catch (error) { - // The source file was already created, so we need to delete it - await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); - const errorDetails = (error as any).details; - - throw { - message: error.message || "Unable to establish a connection", - details: - errorDetails && errorDetails !== error.message - ? errorDetails - : undefined, - }; - } - - // Check for file errors - // If the model file has errors, rollback the changes - const errorMessage = await fileArtifacts.checkFileErrors( - queryClient, - instanceId, - newSourceFilePath, - ); - if (errorMessage) { - await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); - throw new Error(errorMessage); - } - - await goto(`/files/${newSourceFilePath}`); -} - async function saveConnectorAnyway( queryClient: QueryClient, connector: V1ConnectorDriver, @@ -489,3 +407,85 @@ export async function submitAddConnectorForm( // Wait for the submission to complete await submissionPromise; } + +export async function submitAddSourceForm( + queryClient: QueryClient, + connector: V1ConnectorDriver, + formValues: AddDataFormValues, +): Promise { + const instanceId = get(runtime).instanceId; + await beforeSubmitForm(instanceId, connector); + + const newSourceName = formValues.name as string; + + const [rewrittenConnector, rewrittenFormValues] = prepareSourceFormData( + connector, + formValues, + ); + + // Make a new .yaml file + const newSourceFilePath = getFileAPIPathFromNameAndType( + newSourceName, + EntityType.Table, + ); + await runtimeServicePutFile(instanceId, { + path: newSourceFilePath, + blob: compileSourceYAML(rewrittenConnector, rewrittenFormValues), + create: true, + createOnly: false, + }); + + const originalEnvBlob = await getOriginalEnvBlob(queryClient, instanceId); + + // Create or update the `.env` file + const newEnvBlob = await updateDotEnvWithSecrets( + queryClient, + rewrittenConnector, + rewrittenFormValues, + "source", + ); + + // Make sure the file has reconciled before testing the connection + await runtimeServicePutFileAndWaitForReconciliation(instanceId, { + path: ".env", + blob: newEnvBlob, + create: true, + createOnly: false, + }); + + // Wait for source resource-level reconciliation + // This must happen after .env reconciliation since sources depend on secrets + try { + await waitForResourceReconciliation( + instanceId, + newSourceName, + ResourceKind.Model, + ); + } catch (error) { + // The source file was already created, so we need to delete it + await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); + const errorDetails = (error as any).details; + + throw { + message: error.message || "Unable to establish a connection", + details: + errorDetails && errorDetails !== error.message + ? errorDetails + : undefined, + }; + } + + // Check for file errors + // If the model file has errors, rollback the changes + const errorMessage = await fileArtifacts.checkFileErrors( + queryClient, + instanceId, + newSourceFilePath, + ); + if (errorMessage) { + await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); + throw new Error(errorMessage); + } + + await goto(`/files/${newSourceFilePath}`); +} diff --git a/web-common/src/features/sources/modal/types.ts b/web-common/src/features/sources/modal/types.ts index 5cb0785a5a5..8fd405fe6de 100644 --- a/web-common/src/features/sources/modal/types.ts +++ b/web-common/src/features/sources/modal/types.ts @@ -1,3 +1,51 @@ +import type { MultiStepFormSchema } from "../../templates/schemas/types"; + +export type { + JSONSchemaCondition, + JSONSchemaConditional, + JSONSchemaConstraint, + JSONSchemaField, + JSONSchemaObject, + MultiStepFormSchema, +} from "../../templates/schemas/types"; + export type AddDataFormType = "source" | "connector"; export type ConnectorType = "parameters" | "dsn"; + +export type AuthOption = { + value: string; + label: string; + description: string; + hint?: string; +}; + +export type AuthField = + | { + type: "credentials"; + id: string; + hint?: string; + optional?: boolean; + accept?: string; + } + | { + type: "input"; + id: string; + label: string; + placeholder?: string; + optional?: boolean; + secret?: boolean; + hint?: string; + }; + +export type MultiStepFormConfig = { + schema: MultiStepFormSchema; + authMethodKey: string; + authOptions: AuthOption[]; + clearFieldsByMethod: Record; + excludedKeys: string[]; + authFieldGroups: Record; + requiredFieldsByMethod: Record; + fieldLabels: Record; + defaultAuthMethod?: string; +}; diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 5a7128802d2..6bd2b9106f4 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -1,6 +1,13 @@ import { humanReadableErrorMessage } from "../errors/errors"; import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; import type { ClickHouseConnectorType } from "./constants"; +import type { MultiStepFormSchema } from "./types"; +import { + findRadioEnumKey, + getRadioEnumOptions, + getRequiredFieldsByEnumValue, + isStepMatch, +} from "../../templates/schema-utils"; /** * Returns true for undefined, null, empty string, or whitespace-only string. @@ -22,16 +29,6 @@ export function isEmpty(val: any) { * - If input resembles a Zod `_errors` array, returns that. * - Otherwise returns undefined. */ -export function normalizeErrors( - err: any, -): string | string[] | null | undefined { - if (!err) return undefined; - if (Array.isArray(err)) return err; - if (typeof err === "string") return err; - if (err._errors && Array.isArray(err._errors)) return err._errors; - return undefined; -} - /** * Converts unknown error inputs into a unified connector error shape. * - Prefers native Error.message when present @@ -83,6 +80,104 @@ export function hasOnlyDsn( return hasDsn && !hasOthers; } +/** + * Returns true when the active multi-step auth method has missing or invalid + * required fields. Falls back to configured default/first auth method. + */ +export function isMultiStepConnectorDisabled( + schema: MultiStepFormSchema | null, + paramsFormValue: Record, + paramsFormErrors: Record, + step?: "connector" | "source" | string, +) { + if (!schema) return true; + + // For source step, gate on required fields from the JSON schema. + const currentStep = step || (paramsFormValue?.__step as string | undefined); + if (currentStep === "source") { + const required = getRequiredFieldsForStep( + schema, + paramsFormValue, + "source", + ); + if (!required.length) return false; + return !required.every((fieldId) => { + if (!isStepMatch(schema, fieldId, "source")) return true; + const value = paramsFormValue[fieldId]; + const errorsForField = paramsFormErrors[fieldId] as any; + const hasErrors = Boolean(errorsForField?.length); + return !isEmpty(value) && !hasErrors; + }); + } + + const authInfo = getRadioEnumOptions(schema); + const options = authInfo?.options ?? []; + const authKey = authInfo?.key || findRadioEnumKey(schema); + const methodFromForm = + authKey && paramsFormValue?.[authKey] != null + ? String(paramsFormValue[authKey]) + : undefined; + const hasValidFormSelection = options.some( + (opt) => opt.value === methodFromForm, + ); + const method = + (hasValidFormSelection && methodFromForm) || + authInfo?.defaultValue || + options[0]?.value; + + if (!method) return true; + + // Selecting "public" should always enable the button for multi-step auth flows. + if (method === "public") return false; + + const requiredByMethod = getRequiredFieldsByEnumValue(schema, { + step: "connector", + }); + const requiredFields = requiredByMethod[method] ?? []; + if (!requiredFields.length) return true; + + return !requiredFields.every((fieldId) => { + if (!isStepMatch(schema, fieldId, "connector")) return true; + const value = paramsFormValue[fieldId]; + const errorsForField = paramsFormErrors[fieldId] as any; + const hasErrors = Boolean(errorsForField?.length); + return !isEmpty(value) && !hasErrors; + }); +} + +function getRequiredFieldsForStep( + schema: MultiStepFormSchema, + values: Record, + step: "connector" | "source" | string, +) { + const required = new Set(); + (schema.required ?? []).forEach((key) => { + if (isStepMatch(schema, key, step)) required.add(key); + }); + + for (const conditional of schema.allOf ?? []) { + const condition = conditional.if?.properties; + const matches = matchesCondition(condition, values); + const branch = matches ? conditional.then : conditional.else; + branch?.required?.forEach((key) => { + if (isStepMatch(schema, key, step)) required.add(key); + }); + } + + return Array.from(required); +} + +function matchesCondition( + condition: Record | undefined, + values: Record, +) { + if (!condition || !Object.keys(condition).length) return false; + return Object.entries(condition).every(([depKey, def]) => { + if (def.const === undefined || def.const === null) return false; + return String(values?.[depKey]) === String(def.const); + }); +} + /** * Applies ClickHouse Cloud-specific default requirements for connector values. * - For ClickHouse Cloud: enforces `ssl: true` diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index b12ec332eae..18fe8befa14 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -5,28 +5,6 @@ import { } from "../../entity-management/name-utils"; export const getYupSchema = { - s3: yup.object().shape({ - path: yup - .string() - .matches(/^s3:\/\//, "Must be an S3 URI (e.g. s3://bucket/path)") - .required("S3 URI is required"), - aws_region: yup.string(), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), - }), - - gcs: yup.object().shape({ - google_application_credentials: yup.string().optional(), - key_id: yup.string().optional(), - secret: yup.string().optional(), - path: yup - .string() - .matches(/^gs:\/\//, "Must be a GS URI (e.g. gs://bucket/path)") - .optional(), - }), - https: yup.object().shape({ path: yup .string() @@ -65,21 +43,6 @@ export const getYupSchema = { .required("Google application credentials is required"), }), - azure: yup.object().shape({ - path: yup - .string() - .matches( - /^azure:\/\//, - "Must be an Azure URI (e.g. azure://container/path)", - ) - .required("Path is required"), - azure_storage_account: yup.string(), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), - }), - postgres: yup.object().shape({ dsn: yup.string().optional(), host: yup.string().optional(), diff --git a/web-common/src/features/sources/sourceUtils.ts b/web-common/src/features/sources/sourceUtils.ts index 480709142f6..d181415b56e 100644 --- a/web-common/src/features/sources/sourceUtils.ts +++ b/web-common/src/features/sources/sourceUtils.ts @@ -165,6 +165,11 @@ export function maybeRewriteToDuckDb( case "gcs": case "https": case "azure": + // Ensure DuckDB creates a temporary secret for the original connector + if (!formValues.create_secrets_from_connectors) { + formValues.create_secrets_from_connectors = connector.name; + } + // falls through to rewrite as DuckDB case "local_file": connectorCopy.name = "duckdb"; @@ -212,6 +217,9 @@ export function prepareSourceFormData( // Create a copy of form values to avoid mutating the original const processedValues = { ...formValues }; + // Never carry connector auth selection into the source/model layer. + delete processedValues.auth_method; + // Strip connector configuration keys from the source form values to prevent // leaking connector-level fields (e.g., credentials) into the model file. if (connector.configProperties) { diff --git a/web-common/src/features/templates/JSONSchemaFormRenderer.svelte b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte new file mode 100644 index 00000000000..6b6bcfa294b --- /dev/null +++ b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte @@ -0,0 +1,279 @@ + + +
+ {#if schema} + {#each renderOrder as [key, prop]} + {#if isRadioEnum(prop)} +
+ {#if prop.title} +
{prop.title}
+ {/if} + + + {#if groupedFields.get(key)} + {#each getGroupedFieldsForOption(key, option.value) as [childKey, childProp]} +
+ +
+ {/each} + {/if} + + +
+ {:else} +
+ +
+ {/if} + {/each} + {/if} +
diff --git a/web-common/src/features/templates/SchemaField.svelte b/web-common/src/features/templates/SchemaField.svelte new file mode 100644 index 00000000000..490287bb05a --- /dev/null +++ b/web-common/src/features/templates/SchemaField.svelte @@ -0,0 +1,55 @@ + + +{#if prop["x-display"] === "file" || prop.format === "file"} + +{:else if prop.type === "boolean"} + +{:else if options?.length} + +{:else} + onStringInputChange(e)} + alwaysShowError + /> +{/if} diff --git a/web-common/src/features/templates/error-utils.ts b/web-common/src/features/templates/error-utils.ts new file mode 100644 index 00000000000..1c2c74dcbf4 --- /dev/null +++ b/web-common/src/features/templates/error-utils.ts @@ -0,0 +1,16 @@ +/** + * Normalizes a variety of error shapes into a string, string[], or undefined. + * - If input is an array, returns it as-is. + * - If input is a string, returns it. + * - If input resembles a Zod `_errors` array, returns that. + * - Otherwise returns undefined. + */ +export function normalizeErrors( + err: any, +): string | string[] | null | undefined { + if (!err) return undefined; + if (Array.isArray(err)) return err; + if (typeof err === "string") return err; + if (err._errors && Array.isArray(err._errors)) return err._errors; + return undefined; +} diff --git a/web-common/src/features/templates/schema-utils.ts b/web-common/src/features/templates/schema-utils.ts new file mode 100644 index 00000000000..a4544947297 --- /dev/null +++ b/web-common/src/features/templates/schema-utils.ts @@ -0,0 +1,154 @@ +import type { + JSONSchemaConditional, + MultiStepFormSchema, +} from "./schemas/types"; + +export type RadioEnumOption = { + value: string; + label: string; + description: string; + hint?: string; +}; + +export function isStepMatch( + schema: MultiStepFormSchema | null, + key: string, + step?: "connector" | "source" | string, +): boolean { + if (!schema?.properties) return false; + const prop = schema.properties[key]; + if (!prop) return false; + if (!step) return true; + const propStep = prop["x-step"]; + if (!propStep) return true; + return propStep === step; +} + +export function isVisibleForValues( + schema: MultiStepFormSchema, + key: string, + values: Record, +): boolean { + const prop = schema.properties?.[key]; + if (!prop) return false; + const conditions = prop["x-visible-if"]; + if (!conditions) return true; + + return Object.entries(conditions).every(([depKey, expected]) => { + const actual = values?.[depKey]; + if (Array.isArray(expected)) { + return expected.map(String).includes(String(actual)); + } + return String(actual) === String(expected); + }); +} + +export function getFieldLabel( + schema: MultiStepFormSchema, + key: string, +): string { + return schema.properties?.[key]?.title || key; +} + +export function findRadioEnumKey(schema: MultiStepFormSchema): string | null { + if (!schema.properties) return null; + for (const [key, value] of Object.entries(schema.properties)) { + if (value.enum && value["x-display"] === "radio") { + return key; + } + } + return null; +} + +export function getRadioEnumOptions(schema: MultiStepFormSchema): { + key: string; + options: RadioEnumOption[]; + defaultValue?: string; +} | null { + const enumKey = findRadioEnumKey(schema); + if (!enumKey) return null; + const enumProperty = schema.properties?.[enumKey]; + if (!enumProperty?.enum) return null; + + const labels = enumProperty["x-enum-labels"] ?? []; + const descriptions = enumProperty["x-enum-descriptions"] ?? []; + const options = + enumProperty.enum?.map((value, idx) => ({ + value: String(value), + label: labels[idx] ?? String(value), + description: + descriptions[idx] ?? enumProperty.description ?? "Choose an option", + hint: enumProperty["x-hint"], + })) ?? []; + + const defaultValue = + enumProperty.default !== undefined && enumProperty.default !== null + ? String(enumProperty.default) + : options[0]?.value; + + return { + key: enumKey, + options, + defaultValue: defaultValue || undefined, + }; +} + +export function getRequiredFieldsByEnumValue( + schema: MultiStepFormSchema, + opts?: { step?: "connector" | "source" | string }, +): Record { + const enumInfo = getRadioEnumOptions(schema); + if (!enumInfo) return {}; + + const conditionals = schema.allOf ?? []; + const baseRequired = new Set(schema.required ?? []); + const result: Record = {}; + + const matchesStep = (field: string) => { + if (!opts?.step) return true; + const prop = schema.properties?.[field]; + if (!prop) return false; + const propStep = prop["x-step"]; + if (!propStep) return true; + return propStep === opts.step; + }; + + for (const option of enumInfo.options) { + const required = new Set(); + + baseRequired.forEach((field) => { + if (matchesStep(field)) { + required.add(field); + } + }); + + for (const conditional of conditionals) { + const matches = matchesEnumCondition( + conditional, + enumInfo.key, + option.value, + ); + const target = matches ? conditional.then : conditional.else; + target?.required?.forEach((field) => { + if (matchesStep(field)) { + required.add(field); + } + }); + } + + result[option.value] = Array.from(required); + } + + return result; +} + +function matchesEnumCondition( + conditional: JSONSchemaConditional, + enumKey: string, + value: string, +) { + const conditionProps = conditional.if?.properties; + const constValue = conditionProps?.[enumKey]?.const; + if (constValue === undefined || constValue === null) return false; + return String(constValue) === value; +} diff --git a/web-common/src/features/templates/schemas/azure.ts b/web-common/src/features/templates/schemas/azure.ts new file mode 100644 index 00000000000..db741448076 --- /dev/null +++ b/web-common/src/features/templates/schemas/azure.ts @@ -0,0 +1,108 @@ +import type { MultiStepFormSchema } from "./types"; + +export const azureSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["connection_string", "account_key", "sas_token", "public"], + default: "connection_string", + description: "Choose how to authenticate to Azure Blob Storage", + "x-display": "radio", + "x-enum-labels": [ + "Connection String", + "Storage Account Key", + "SAS Token", + "Public", + ], + "x-enum-descriptions": [ + "Provide a full Azure Storage connection string.", + "Provide the storage account name and access key.", + "Provide the storage account name and SAS token.", + "Access publicly readable blobs without credentials.", + ], + "x-grouped-fields": { + connection_string: ["azure_storage_connection_string"], + account_key: ["azure_storage_account", "azure_storage_key"], + sas_token: ["azure_storage_account", "azure_storage_sas_token"], + public: [], + }, + "x-step": "connector", + }, + azure_storage_connection_string: { + type: "string", + title: "Connection string", + description: "Paste an Azure Storage connection string", + "x-placeholder": "Enter Azure storage connection string", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "connection_string" }, + }, + azure_storage_account: { + type: "string", + title: "Storage account", + description: "The name of the Azure storage account", + "x-placeholder": "Enter Azure storage account", + "x-step": "connector", + "x-visible-if": { auth_method: ["account_key", "sas_token"] }, + }, + azure_storage_key: { + type: "string", + title: "Access key", + description: "Primary or secondary access key for the storage account", + "x-placeholder": "Enter Azure storage access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "account_key" }, + }, + azure_storage_sas_token: { + type: "string", + title: "SAS token", + description: + "Shared Access Signature token for the storage account (starting with ?sv=)", + "x-placeholder": "Enter Azure SAS token", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "sas_token" }, + }, + path: { + type: "string", + title: "Blob URI", + description: + "URI to the Azure blob container or directory (e.g., https://.blob.core.windows.net/container)", + pattern: "^azure://.+", + errorMessage: { + pattern: "Must be an Azure URI (e.g. azure://container/path)", + }, + "x-placeholder": "azure://container/path", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z0-9_]+$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + required: ["path", "name"], + allOf: [ + { + if: { properties: { auth_method: { const: "connection_string" } } }, + then: { required: ["azure_storage_connection_string"] }, + }, + { + if: { properties: { auth_method: { const: "account_key" } } }, + then: { required: ["azure_storage_account", "azure_storage_key"] }, + }, + { + if: { properties: { auth_method: { const: "sas_token" } } }, + then: { + required: ["azure_storage_account", "azure_storage_sas_token"], + }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/gcs.ts b/web-common/src/features/templates/schemas/gcs.ts new file mode 100644 index 00000000000..a4e1125f1ff --- /dev/null +++ b/web-common/src/features/templates/schemas/gcs.ts @@ -0,0 +1,87 @@ +import type { MultiStepFormSchema } from "./types"; + +export const gcsSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["credentials", "hmac", "public"], + default: "credentials", + description: "Choose how to authenticate to GCS", + "x-display": "radio", + "x-enum-labels": ["GCP credentials", "HMAC keys", "Public"], + "x-enum-descriptions": [ + "Upload a JSON key file for a service account with GCS access.", + "Use HMAC access key and secret for S3-compatible authentication.", + "Access publicly readable buckets without credentials.", + ], + "x-grouped-fields": { + credentials: ["google_application_credentials"], + hmac: ["key_id", "secret"], + public: [], + }, + "x-step": "connector", + }, + google_application_credentials: { + type: "string", + title: "Service account key", + description: + "Upload a JSON key file for a service account with GCS access.", + format: "file", + "x-display": "file", + "x-accept": ".json", + "x-step": "connector", + "x-visible-if": { auth_method: "credentials" }, + }, + key_id: { + type: "string", + title: "Access Key ID", + description: "HMAC access key ID for S3-compatible authentication", + "x-placeholder": "Enter your HMAC access key ID", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "hmac" }, + }, + secret: { + type: "string", + title: "Secret Access Key", + description: "HMAC secret access key for S3-compatible authentication", + "x-placeholder": "Enter your HMAC secret access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "hmac" }, + }, + path: { + type: "string", + title: "GCS URI", + description: "Path to your GCS bucket or prefix", + pattern: "^gs://[^/]+(/.*)?$", + errorMessage: { + pattern: "Must be a GS URI (e.g. gs://bucket/path)", + }, + "x-placeholder": "gs://bucket/path", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z0-9_]+$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + required: ["path", "name"], + allOf: [ + { + if: { properties: { auth_method: { const: "credentials" } } }, + then: { required: ["google_application_credentials"] }, + }, + { + if: { properties: { auth_method: { const: "hmac" } } }, + then: { required: ["key_id", "secret"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/s3.ts b/web-common/src/features/templates/schemas/s3.ts new file mode 100644 index 00000000000..ea19e1f6f5a --- /dev/null +++ b/web-common/src/features/templates/schemas/s3.ts @@ -0,0 +1,105 @@ +import type { MultiStepFormSchema } from "./types"; + +export const s3Schema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + description: "Choose how to authenticate to S3", + enum: ["access_keys", "public"], + default: "access_keys", + "x-display": "radio", + "x-enum-labels": ["Access keys", "Public"], + "x-enum-descriptions": [ + "Use AWS access key ID and secret access key.", + "Access publicly readable buckets without credentials.", + ], + "x-grouped-fields": { + access_keys: [ + "aws_access_key_id", + "aws_secret_access_key", + "region", + "endpoint", + "aws_role_arn", + ], + public: [], + }, + "x-step": "connector", + }, + aws_access_key_id: { + type: "string", + title: "Access Key ID", + description: "AWS access key ID for the bucket", + "x-placeholder": "Enter AWS access key ID", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + aws_secret_access_key: { + type: "string", + title: "Secret Access Key", + description: "AWS secret access key for the bucket", + "x-placeholder": "Enter AWS secret access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + region: { + type: "string", + title: "Region", + description: + "Rill uses your default AWS region unless you set it explicitly.", + "x-placeholder": "us-east-1", + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + endpoint: { + type: "string", + title: "Endpoint", + description: + "Override the S3 endpoint (for S3-compatible services like R2/MinIO).", + "x-placeholder": "https://s3.example.com", + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + aws_role_arn: { + type: "string", + title: "AWS Role ARN", + description: "AWS Role ARN to assume", + "x-placeholder": "arn:aws:iam::123456789012:role/MyRole", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + path: { + type: "string", + title: "S3 URI", + description: "Path to your S3 bucket or prefix", + pattern: "^s3://[^/]+(/.*)?$", + errorMessage: { + pattern: "Must be an S3 URI (e.g. s3://bucket/path)", + }, + "x-placeholder": "s3://bucket/path", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z0-9_]+$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + required: ["path", "name"], + allOf: [ + { + if: { properties: { auth_method: { const: "access_keys" } } }, + then: { + required: ["aws_access_key_id", "aws_secret_access_key"], + }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/types.ts b/web-common/src/features/templates/schemas/types.ts new file mode 100644 index 00000000000..d8ccd6b11b9 --- /dev/null +++ b/web-common/src/features/templates/schemas/types.ts @@ -0,0 +1,64 @@ +export type JSONSchemaVisibleIfValue = + | string + | number + | boolean + | Array; + +export type JSONSchemaField = { + type?: "string" | "number" | "boolean" | "object"; + title?: string; + description?: string; + enum?: Array; + const?: string | number | boolean; + default?: string | number | boolean; + pattern?: string; + format?: string; + errorMessage?: { + pattern?: string; + format?: string; + }; + properties?: Record; + required?: string[]; + "x-display"?: "radio" | "select" | "textarea" | "file"; + "x-step"?: "connector" | "source"; + "x-secret"?: boolean; + "x-visible-if"?: Record; + "x-enum-labels"?: string[]; + "x-enum-descriptions"?: string[]; + "x-placeholder"?: string; + "x-hint"?: string; + "x-accept"?: string; + /** + * Explicit grouping for radio/select options: maps an option value to the + * child field keys that should render beneath that option. + */ + "x-grouped-fields"?: Record; + // Allow custom keywords such as errorMessage or future x-extensions. + [key: string]: unknown; +}; + +export type JSONSchemaCondition = { + properties?: Record; +}; + +export type JSONSchemaConstraint = { + required?: string[]; +}; + +export type JSONSchemaConditional = { + if?: JSONSchemaCondition; + then?: JSONSchemaConstraint; + else?: JSONSchemaConstraint; +}; + +export type JSONSchemaObject = { + $schema?: string; + type: "object"; + title?: string; + description?: string; + properties?: Record; + required?: string[]; + allOf?: JSONSchemaConditional[]; +}; + +export type MultiStepFormSchema = JSONSchemaObject; diff --git a/web-local/tests/connectors/azure-connector.spec.ts b/web-local/tests/connectors/azure-connector.spec.ts new file mode 100644 index 00000000000..a4b19724b6c --- /dev/null +++ b/web-local/tests/connectors/azure-connector.spec.ts @@ -0,0 +1,44 @@ +import { expect } from "@playwright/test"; +import { test } from "../setup/base"; + +test.describe("Azure connector form reset", () => { + test.use({ project: "Blank" }); + + test("clears connection string after submit and reopen", async ({ page }) => { + // Open Add Data modal and pick Azure. + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + await page.locator("#azure").click(); + await page.waitForSelector('form[id*="azure"]'); + + const connectionString = page.getByRole("textbox", { + name: "Connection string", + }); + const submit = page + .getByRole("dialog") + .getByRole("button", { name: /(Test and Connect|Continue)/ }); + + // Fill a connection string so the CTA enables and submit once. + await connectionString.fill( + "DefaultEndpointsProtocol=https;AccountName=test;AccountKey=abc;", + ); + await expect(submit).toBeEnabled(); + await submit.click(); + + // Save Anyway (expected for failed test) to close the modal, then wait for form unmount. + const saveAnyway = page + .getByRole("dialog") + .getByRole("button", { name: "Save Anyway" }); + await expect(saveAnyway).toBeVisible(); + await saveAnyway.click(); + await page.waitForSelector('form[id*="azure"]', { state: "detached" }); + + // Re-open and ensure the connection string is cleared. + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + await page.locator("#azure").click(); + await page.waitForSelector('form[id*="azure"]'); + + await expect(connectionString).toHaveValue(""); + }); +}); diff --git a/web-local/tests/connectors/multi-step-connector.spec.ts b/web-local/tests/connectors/multi-step-connector.spec.ts new file mode 100644 index 00000000000..09e962d87b4 --- /dev/null +++ b/web-local/tests/connectors/multi-step-connector.spec.ts @@ -0,0 +1,225 @@ +import { expect } from "@playwright/test"; +import { test } from "../setup/base"; + +test.describe("Multi-step connector wrapper", () => { + test.use({ project: "Blank" }); + + test("GCS connector - renders connector step schema via wrapper", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + + // Choose a multi-step connector (GCS). + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + + // Connector step should show connector preview and connector CTA. + await expect(page.getByText("Connector preview")).toBeVisible(); + await expect( + page + .getByRole("dialog") + .getByRole("button", { name: "Test and Connect" }), + ).toBeVisible(); + + // Auth method controls from the connector schema should render. + const hmacRadio = page.getByRole("radio", { name: "HMAC keys" }); + await expect(hmacRadio).toBeVisible(); + await expect(page.getByRole("radio", { name: "Public" })).toBeVisible(); + + // Select HMAC so its fields are rendered. + await hmacRadio.click(); + + // Connector step fields should be present, while source step fields should not yet render. + await expect( + page.getByRole("textbox", { name: "Access Key ID" }), + ).toBeVisible(); + await expect( + page.getByRole("textbox", { name: "Secret Access Key" }), + ).toBeVisible(); + await expect(page.getByRole("textbox", { name: "GS URI" })).toHaveCount(0); + }); + + test("GCS connector - renders source step schema via wrapper", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + + // Choose a multi-step connector (GCS). + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + + // Connector step visible with CTA. + await expect(page.getByText("Connector preview")).toBeVisible(); + await expect( + page + .getByRole("dialog") + .getByRole("button", { name: "Test and Connect" }), + ).toBeVisible(); + + // Switch to Public auth (no required fields) and continue via CTA. + await page.getByRole("radio", { name: "Public" }).click(); + const connectorCta = page.getByRole("button", { + name: /Test and Connect|Continue/i, + }); + await connectorCta.click(); + + // Source step should now render with source schema fields and CTA. + await expect(page.getByText("Model preview")).toBeVisible(); + const sourceCta = page.getByRole("button", { + name: /Test and Add data|Importing data|Add data/i, + }); + await expect(sourceCta).toBeVisible(); + + // Source fields should be present; connector-only auth fields should not be required to show. + await expect(page.getByRole("textbox", { name: "GCS URI" })).toBeVisible( + {}, + ); + await expect( + page.getByRole("textbox", { name: "Model name" }), + ).toBeVisible(); + }); + + test("GCS connector - preserves auth selection across steps", async ({ + page, + }) => { + const hmacKey = process.env.RILL_RUNTIME_GCS_TEST_HMAC_KEY; + const hmacSecret = process.env.RILL_RUNTIME_GCS_TEST_HMAC_SECRET; + if (!hmacKey || !hmacSecret) { + test.skip( + true, + "RILL_RUNTIME_GCS_TEST_HMAC_KEY or RILL_RUNTIME_GCS_TEST_HMAC_SECRET is not set", + ); + } + + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + + // Pick HMAC auth and fill required fields. + await page.getByRole("radio", { name: "HMAC keys" }).click(); + await page.getByRole("textbox", { name: "Access Key ID" }).fill(hmacKey!); + await page + .getByRole("textbox", { name: "Secret Access Key" }) + .fill(hmacSecret!); + + // Submit connector step via CTA to transition to source step. + const connectorCta = page.getByRole("button", { + name: /Test and Connect|Continue/i, + }); + await expect(connectorCta).toBeEnabled(); + await connectorCta.click(); + await expect(page.getByText("Model preview")).toBeVisible(); + + // Go back to connector step. + await page.getByRole("button", { name: "Back" }).click(); + + // Auth selection and values should persist. + await expect(page.getByText("Connector preview")).toBeVisible(); + await expect(page.getByRole("radio", { name: "HMAC keys" })).toBeChecked({ + timeout: 5000, + }); + await expect( + page.getByRole("textbox", { name: "Access Key ID" }), + ).toHaveValue(hmacKey!); + await expect( + page.getByRole("textbox", { name: "Secret Access Key" }), + ).toHaveValue(hmacSecret!); + }); + + test("GCS connector - disables submit until auth requirements met", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + + const connectorCta = page.getByRole("button", { + name: /Test and Connect|Continue/i, + }); + + // Default auth is credentials (file upload); switch to HMAC to check required fields. + await page.getByRole("radio", { name: "HMAC keys" }).click(); + await expect(connectorCta).toBeDisabled(); + + // Fill key only -> still disabled. + await page + .getByRole("textbox", { name: "Access Key ID" }) + .fill("AKIA_TEST"); + await expect(connectorCta).toBeDisabled(); + + // Fill secret -> enabled. + await page + .getByRole("textbox", { name: "Secret Access Key" }) + .fill("SECRET"); + await expect(connectorCta).toBeEnabled(); + }); + + test("GCS connector - public auth option keeps submit enabled and allows advancing", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + + const connectorCta = page.getByRole("button", { + name: /Test and Connect|Continue/i, + }); + + // Switch to Public (no required fields) -> CTA should remain enabled and allow advancing. + await page.getByRole("radio", { name: "Public" }).click(); + await expect(connectorCta).toBeEnabled(); + await connectorCta.click(); + + // Should land on source step without needing connector fields. + await expect(page.getByText("Model preview")).toBeVisible(); + await expect( + page.getByRole("button", { name: /Test and Add data|Add data/i }), + ).toBeVisible(); + }); + + test("GCS connector - Save Anyway cleared when advancing to model step after HMAC", async ({ + page, + }) => { + // Skip test if environment variables are not set + if ( + !process.env.RILL_RUNTIME_GCS_TEST_HMAC_KEY || + !process.env.RILL_RUNTIME_GCS_TEST_HMAC_SECRET + ) { + test.skip( + true, + "RILL_RUNTIME_GCS_TEST_HMAC_KEY or RILL_RUNTIME_GCS_TEST_HMAC_SECRET environment variable is not set", + ); + } + + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + + // Trigger Save Anyway on connector step by failing Test and Connect with HMAC. + await page.getByRole("radio", { name: "HMAC keys" }).click(); + await page + .getByRole("textbox", { name: "Access Key ID" }) + .fill(process.env.RILL_RUNTIME_GCS_TEST_HMAC_KEY!); + await page + .getByRole("textbox", { name: "Secret Access Key" }) + .fill(process.env.RILL_RUNTIME_GCS_TEST_HMAC_SECRET!); + + await page + .getByRole("dialog") + .getByRole("button", { name: "Test and Connect" }) + .click(); + + const saveAnywayButton = page.getByRole("button", { name: "Save Anyway" }); + await expect(saveAnywayButton).toBeHidden(); + }); +}); diff --git a/web-local/tests/connectors/test-connection.spec.ts b/web-local/tests/connectors/test-connection.spec.ts index e571c62d3bf..1ed49d8276c 100644 --- a/web-local/tests/connectors/test-connection.spec.ts +++ b/web-local/tests/connectors/test-connection.spec.ts @@ -4,6 +4,73 @@ import { test } from "../setup/base"; test.describe("Test Connection", () => { test.use({ project: "Blank" }); + test("Azure connector - auth method specific required fields", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + await page.locator("#azure").click(); + await page.waitForSelector('form[id*="azure"]'); + + const button = page.getByRole("dialog").getByRole("button", { + name: /(Test and Connect|Continue)/, + }); + + // Select Storage Account Key (default may be different) -> requires account + key. + await page.getByRole("radio", { name: "Storage Account Key" }).click(); + await expect(button).toBeDisabled(); + + await page.getByRole("textbox", { name: "Storage account" }).fill("acct"); + await expect(button).toBeDisabled(); + + await page.getByRole("textbox", { name: "Access key" }).fill("key"); + await expect(button).toBeEnabled(); + + // Switch to Public (no required fields) -> button should stay enabled. + await page.getByRole("radio", { name: "Public" }).click(); + await expect(button).toBeEnabled(); + + // Switch to Connection String -> requires connection string, so disabled until filled. + await page.getByRole("radio", { name: "Connection String" }).click(); + await expect(button).toBeDisabled(); + await page + .getByRole("textbox", { name: "Connection string" }) + .fill("DefaultEndpointsProtocol=https;"); + await expect(button).toBeEnabled(); + }); + + test("S3 connector - auth method specific required fields", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + await page.locator("#s3").click(); + await page.waitForSelector('form[id*="s3"]'); + + const button = page.getByRole("dialog").getByRole("button", { + name: /(Test and Connect|Continue)/, + }); + + // Default method is Access keys -> requires access key id + secret. + await expect(button).toBeDisabled(); + await page + .getByRole("textbox", { name: "Access Key ID" }) + .fill("AKIA_TEST"); + await expect(button).toBeDisabled(); + await page + .getByRole("textbox", { name: "Secret Access Key" }) + .fill("SECRET"); + await expect(button).toBeEnabled(); + + // Switch to Public (no required fields) -> button should stay enabled. + await page.getByRole("radio", { name: "Public" }).click(); + await expect(button).toBeEnabled(); + + // Switch back to Access keys -> fields cleared, so disabled until refilled. + await page.getByRole("radio", { name: "Access keys" }).click(); + await expect(button).toBeDisabled(); + }); + test("GCS connector - HMAC", async ({ page }) => { // Skip test if environment variables are not set if (