diff --git a/web-common/src/components/forms/Input.svelte b/web-common/src/components/forms/Input.svelte index 89b14da5992..78395ab6705 100644 --- a/web-common/src/components/forms/Input.svelte +++ b/web-common/src/components/forms/Input.svelte @@ -166,7 +166,8 @@ contenteditable class="multiline-input" class:pointer-events-none={disabled} - {placeholder} + data-placeholder={placeholder} + aria-label={label || title || placeholder} role="textbox" tabindex="0" aria-multiline="true" @@ -327,6 +328,11 @@ word-wrap: break-word; } + .multiline-input:empty:before { + content: attr(data-placeholder); + @apply text-gray-400; + } + .input-wrapper:focus-within { @apply border-primary-500; @apply ring-2 ring-primary-100; diff --git a/web-common/src/features/connectors/code-utils.ts b/web-common/src/features/connectors/code-utils.ts index edb0bcb4e06..dd1b8f7711a 100644 --- a/web-common/src/features/connectors/code-utils.ts +++ b/web-common/src/features/connectors/code-utils.ts @@ -1,7 +1,6 @@ import { QueryClient } from "@tanstack/svelte-query"; import { get } from "svelte/store"; import { - ConnectorDriverPropertyType, type V1ConnectorDriver, type ConnectorDriverProperty, getRuntimeServiceGetFileQueryKey, @@ -37,9 +36,18 @@ export function compileConnectorYAML( connector: V1ConnectorDriver, formValues: Record, options?: { - fieldFilter?: (property: ConnectorDriverProperty) => boolean; - orderedProperties?: ConnectorDriverProperty[]; + fieldFilter?: ( + property: + | ConnectorDriverProperty + | { key?: string; type?: string; secret?: boolean; internal?: boolean }, + ) => boolean; + orderedProperties?: Array< + | ConnectorDriverProperty + | { key?: string; type?: string; secret?: boolean } + >; connectorInstanceName?: string; + secretKeys?: string[]; + stringKeys?: string[]; }, ) { // Add instructions to the top of the file @@ -50,12 +58,8 @@ type: connector driver: ${getDriverNameForConnector(connector.name as string)}`; - // Use the provided orderedProperties if available, otherwise fall back to configProperties/sourceProperties - let properties = - options?.orderedProperties ?? - connector.configProperties ?? - connector.sourceProperties ?? - []; + // Use the provided orderedProperties if available. + let properties = options?.orderedProperties ?? []; // Optionally filter properties if (options?.fieldFilter) { @@ -63,18 +67,10 @@ driver: ${getDriverNameForConnector(connector.name as string)}`; } // Get the secret property keys - const secretPropertyKeys = - connector.configProperties - ?.filter((property) => property.secret) - .map((property) => property.key) || []; + const secretPropertyKeys = options?.secretKeys ?? []; // Get the string property keys - const stringPropertyKeys = - connector.configProperties - ?.filter( - (property) => property.type === ConnectorDriverPropertyType.TYPE_STRING, - ) - .map((property) => property.key) || []; + const stringPropertyKeys = options?.stringKeys ?? []; // Compile key value pairs in the order of properties const compiledKeyValues = properties @@ -100,11 +96,12 @@ driver: ${getDriverNameForConnector(connector.name as string)}`; const isSecretProperty = secretPropertyKeys.includes(key); if (isSecretProperty) { - return `${key}: "{{ .env.${makeDotEnvConnectorKey( + const placeholder = `{{ .env.${makeDotEnvConnectorKey( connector.name as string, key, options?.connectorInstanceName, - )} }}"`; + )} }}`; + return `${key}: "${placeholder}"`; } const isStringProperty = stringPropertyKeys.includes(key); @@ -126,6 +123,7 @@ export async function updateDotEnvWithSecrets( formValues: Record, formType: "source" | "connector", connectorInstanceName?: string, + opts?: { secretKeys?: string[] }, ): Promise { const instanceId = get(runtime).instanceId; @@ -147,13 +145,7 @@ export async function updateDotEnvWithSecrets( } // Get the secret keys - const properties = - formType === "source" - ? connector.sourceProperties - : connector.configProperties; - const secretKeys = properties - ?.filter((property) => property.secret) - .map((property) => property.key); + const secretKeys = opts?.secretKeys ?? []; // In reality, all connectors have secret keys, but this is a safeguard if (!secretKeys) { diff --git a/web-common/src/features/sources/modal/AddClickHouseForm.svelte b/web-common/src/features/sources/modal/AddClickHouseForm.svelte deleted file mode 100644 index 5aa3a74d6ee..00000000000 --- a/web-common/src/features/sources/modal/AddClickHouseForm.svelte +++ /dev/null @@ -1,507 +0,0 @@ - - -
-
- - {#if connectorType === "rill-managed"} -
- -
- {/if} -
- - {#if connectorType === "self-hosted" || connectorType === "clickhouse-cloud"} - - -
- {#each filteredProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} - {@const isPortField = propertyKey === "port"} - {@const isSSLField = propertyKey === "ssl"} - -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - options={connectorType === "clickhouse-cloud" && isPortField - ? [ - { value: "8443", label: "8443 (HTTPS)" }, - { value: "9440", label: "9440 (Native Secure)" }, - ] - : undefined} - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {/if} -
- {/each} -
- - -
- {#each dsnProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} -
- -
- {/each} -
- - - {:else} - -
- {#each filteredProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {/if} -
- {/each} -
- {/if} -
diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index bfbffa2efe8..c2433194b99 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -3,32 +3,35 @@ import SubmissionError from "@rilldata/web-common/components/forms/SubmissionError.svelte"; import { queryClient } from "@rilldata/web-common/lib/svelte-query/globalQueryClient"; - import { - type ConnectorDriverProperty, - type V1ConnectorDriver, - } from "@rilldata/web-common/runtime-client"; + import { type V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; import type { ActionResult } from "@sveltejs/kit"; import type { SuperValidated } from "sveltekit-superforms"; import type { AddDataFormType, ConnectorType } from "./types"; - import AddClickHouseForm from "./AddClickHouseForm.svelte"; - import ConnectorForm from "./ConnectorForm.svelte"; + import MultiStepConnectorFlow from "./MultiStepConnectorFlow.svelte"; import NeedHelpText from "./NeedHelpText.svelte"; import Tabs from "@rilldata/web-common/components/forms/Tabs.svelte"; import { TabsContent } from "@rilldata/web-common/components/tabs"; import { hasOnlyDsn, isEmpty } from "./utils"; + import JSONSchemaFormRenderer from "../../templates/JSONSchemaFormRenderer.svelte"; import { CONNECTION_TAB_OPTIONS, type ClickHouseConnectorType, - FORM_HEIGHT_DEFAULT, - MULTI_STEP_CONNECTORS, } from "./constants"; - - import FormRenderer from "./FormRenderer.svelte"; + import { connectorStepStore } from "./connectorStepStore"; import YamlPreview from "./YamlPreview.svelte"; - - import { AddDataFormManager } from "./AddDataFormManager"; + import { + AddDataFormManager, + type ClickhouseUiState, + } from "./AddDataFormManager"; import AddDataFormSection from "./AddDataFormSection.svelte"; + import { get } from "svelte/store"; + import { getConnectorSchema } from "./connector-schemas"; + import { propertiesToSchema } from "./properties-to-schema"; + import { + getRequiredFieldsForValues, + isVisibleForValues, + } from "../../templates/schema-utils"; export let connector: V1ConnectorDriver; export let formType: AddDataFormType; @@ -36,14 +39,9 @@ export let onBack: () => void; export let onClose: () => void; - const isMultiStepConnector = - formType === "connector" && - MULTI_STEP_CONNECTORS.includes(connector.name ?? ""); - let saveAnyway = false; let showSaveAnyway = false; let connectionTab: ConnectorType = "parameters"; - let formHeight = FORM_HEIGHT_DEFAULT; // Wire manager-provided onUpdate after declaration below let handleOnUpdate: < @@ -57,123 +55,182 @@ result: Extract; }) => Promise = async (_event) => {}; - let formManager: AddDataFormManager | null = null; - - $: if (!isMultiStepConnector) { - formManager = new AddDataFormManager({ - connector, - formType, - onParamsUpdate: (e: any) => handleOnUpdate(e), - onDsnUpdate: (e: any) => handleOnUpdate(e), - getSelectedAuthMethod: () => undefined, - }); - formHeight = formManager.formHeight; - } - - let isSourceForm = formType === "source"; - let isConnectorForm = formType === "connector"; - let onlyDsn = false; + const formManager = new AddDataFormManager({ + connector, + formType, + onParamsUpdate: (e: any) => handleOnUpdate(e), + onDsnUpdate: (e: any) => handleOnUpdate(e), + getSelectedAuthMethod: () => + get(connectorStepStore).selectedAuthMethod ?? undefined, + }); + + const isMultiStepConnector = formManager.isMultiStepConnector; + const isExplorerConnector = formManager.isExplorerConnector; + const isStepFlowConnector = isMultiStepConnector || isExplorerConnector; + const isSourceForm = formManager.isSourceForm; + const isConnectorForm = formManager.isConnectorForm; + const onlyDsn = hasOnlyDsn(connector, isConnectorForm); let activeAuthMethod: string | null = null; let prevAuthMethod: string | null = null; + let stepState = $connectorStepStore; let multiStepSubmitDisabled = false; let multiStepButtonLabel = ""; let multiStepLoadingCopy = ""; + let shouldShowSkipLink = false; let primaryButtonLabel = ""; let primaryLoadingCopy = ""; - let shouldShowSkipLink = false; - let multiStepYamlPreview = ""; - let multiStepYamlPreviewTitle = "Connector preview"; - let multiStepSubmitting = false; - let multiStepShowSaveAnyway = false; - let multiStepSaveAnywayLoading = false; - let multiStepSaveAnywayHandler: () => Promise = async () => {}; - let multiStepHandleBack: () => void = () => onBack(); - let multiStepHandleSkip: () => void = () => {}; - - // Form 1: Individual parameters (non-multi-step) - let paramsFormId = ""; - let properties: ConnectorDriverProperty[] = []; - let filteredParamsProperties: ConnectorDriverProperty[] = []; - let multiStepFormId = ""; - let paramsForm: any = null; - let paramsErrors: any = null; - let paramsEnhance: any = null; - let paramsTainted: any = null; - let paramsSubmit: any = null; - let paramsSubmitting: any = null; + + $: stepState = $connectorStepStore; + + // Form 1: Individual parameters + const paramsFormId = formManager.paramsFormId; + const properties = formManager.properties; + const filteredParamsProperties = formManager.filteredParamsProperties; + let multiStepFormId = paramsFormId; + const { + form: paramsForm, + errors: paramsErrors, + enhance: paramsEnhance, + tainted: paramsTainted, + submit: paramsSubmit, + submitting: paramsSubmitting, + } = formManager.params; let paramsError: string | null = null; let paramsErrorDetails: string | undefined = undefined; - // Form 2: DSN (non-multi-step) - let hasDsnFormOption = false; - let dsnFormId = ""; - let dsnProperties: ConnectorDriverProperty[] = []; - let filteredDsnProperties: ConnectorDriverProperty[] = []; - let dsnForm: any = null; - let dsnErrors: any = null; - let dsnEnhance: any = null; - let dsnTainted: any = null; - let dsnSubmit: any = null; - let dsnSubmitting: any = null; + // Form 2: DSN + // SuperForms are not meant to have dynamic schemas, so we use a different form instance for the DSN form + const hasDsnFormOption = formManager.hasDsnFormOption; + const dsnFormId = formManager.dsnFormId; + const dsnProperties = formManager.dsnProperties; + const filteredDsnProperties = formManager.filteredDsnProperties; + const { + form: dsnForm, + errors: dsnErrors, + enhance: dsnEnhance, + tainted: dsnTainted, + submit: dsnSubmit, + submitting: dsnSubmitting, + } = formManager.dsn; let dsnError: string | null = null; let dsnErrorDetails: string | undefined = undefined; - $: if (formManager) { - paramsFormId = formManager.paramsFormId; - properties = formManager.properties; - filteredParamsProperties = formManager.filteredParamsProperties; - multiStepFormId = paramsFormId; - ({ - form: paramsForm, - errors: paramsErrors, - enhance: paramsEnhance, - tainted: paramsTainted, - submit: paramsSubmit, - submitting: paramsSubmitting, - } = formManager.params); - - hasDsnFormOption = formManager.hasDsnFormOption; - dsnFormId = formManager.dsnFormId; - dsnProperties = formManager.dsnProperties; - filteredDsnProperties = formManager.filteredDsnProperties; - ({ - form: dsnForm, - errors: dsnErrors, - enhance: dsnEnhance, - tainted: dsnTainted, - submit: dsnSubmit, - submitting: dsnSubmitting, - } = formManager.dsn); - - isSourceForm = formManager.isSourceForm; - isConnectorForm = formManager.isConnectorForm; - hasDsnFormOption = formManager.hasDsnFormOption; - onlyDsn = hasOnlyDsn(connector, isConnectorForm); - } else { - isSourceForm = formType === "source"; - isConnectorForm = formType === "connector"; - hasDsnFormOption = false; - onlyDsn = false; - } - let clickhouseError: string | null = null; let clickhouseErrorDetails: string | undefined = undefined; - let clickhouseFormId: string = ""; - let clickhouseSubmitting: boolean; - let clickhouseIsSubmitDisabled: boolean; let clickhouseConnectorType: ClickHouseConnectorType = "self-hosted"; - let clickhouseParamsForm; - let clickhouseDsnForm; - let clickhouseShowSaveAnyway: boolean = false; + let prevClickhouseConnectorType: ClickHouseConnectorType | null = null; + let clickhouseUiState: ClickhouseUiState | null = null; + let clickhouseSaving = false; + let effectiveClickhouseSubmitting = false; + + const connectorSchema = getConnectorSchema(connector.name ?? ""); + const hasSchema = Boolean(connectorSchema); + const paramsSchema = + connectorSchema ?? + propertiesToSchema( + filteredParamsProperties as any, + isConnectorForm ? "connector" : "source", + ); + const dsnSchema = + connectorSchema ?? + propertiesToSchema( + filteredDsnProperties as any, + isConnectorForm ? "connector" : "source", + ); + const usesLegacyTabs = !hasSchema && hasDsnFormOption; + $: useDsnForm = usesLegacyTabs && (onlyDsn || connectionTab === "dsn"); + + $: if (connector.name === "clickhouse") { + const nextType = ($paramsForm?.connector_type ?? + clickhouseConnectorType) as ClickHouseConnectorType; + if (nextType && nextType !== clickhouseConnectorType) { + clickhouseConnectorType = nextType; + } + const nextTab = $paramsForm?.connection_mode as ConnectorType | undefined; + if ( + (nextTab === "parameters" || nextTab === "dsn") && + nextTab !== connectionTab + ) { + connectionTab = nextTab; + } + } + + $: if ( + connector.name === "clickhouse" && + clickhouseConnectorType && + clickhouseConnectorType !== prevClickhouseConnectorType + ) { + const defaults = formManager.getClickhouseDefaults(clickhouseConnectorType); + if (defaults) { + paramsForm.update(() => defaults, { taint: false } as any); + } + prevClickhouseConnectorType = clickhouseConnectorType; + } + + // ClickHouse-specific derived state handled by the manager + $: if (connector.name === "clickhouse") { + clickhouseUiState = formManager.computeClickhouseState({ + connectorType: clickhouseConnectorType, + connectionTab, + paramsFormValues: $paramsForm, + dsnFormValues: $paramsForm, + paramsErrors: $paramsErrors, + dsnErrors: $paramsErrors, + paramsForm, + dsnForm: paramsForm, + paramsSubmitting: $paramsSubmitting, + dsnSubmitting: $paramsSubmitting, + }); + + if ( + clickhouseUiState?.enforcedConnectionTab && + clickhouseUiState.enforcedConnectionTab !== connectionTab + ) { + connectionTab = clickhouseUiState.enforcedConnectionTab; + } + + if (clickhouseUiState?.shouldClearErrors) { + clickhouseError = null; + clickhouseErrorDetails = undefined; + } + } else { + clickhouseUiState = null; + } + + $: effectiveClickhouseSubmitting = + connector.name === "clickhouse" + ? clickhouseSaving || clickhouseUiState?.submitting || false + : submitting; + + // Hide Save Anyway once we advance to the model step in step flow connectors. + $: if ( + isStepFlowConnector && + (stepState.step === "source" || stepState.step === "explorer") + ) { + showSaveAnyway = false; + } $: isSubmitDisabled = (() => { - if (isMultiStepConnector) { + if (isStepFlowConnector) { return multiStepSubmitDisabled; } - if (!formManager) return true; + if (connectorSchema) { + const requiredFields = getRequiredFieldsForValues( + connectorSchema, + $paramsForm, + isConnectorForm ? "connector" : "source", + ); + for (const field of requiredFields) { + if (!isVisibleForValues(connectorSchema, field, $paramsForm)) continue; + const value = $paramsForm[field]; + const errorsForField = $paramsErrors[field] as any; + if (isEmpty(value) || errorsForField?.length) return true; + } + return false; + } - if (onlyDsn || connectionTab === "dsn") { + if (useDsnForm) { // DSN form: check required DSN properties for (const property of dsnProperties) { const key = String(property.key); @@ -204,36 +261,34 @@ } })(); - $: formId = isMultiStepConnector - ? multiStepFormId - : (formManager?.getActiveFormId({ connectionTab, onlyDsn }) ?? ""); + $: formId = isStepFlowConnector + ? multiStepFormId || paramsFormId + : useDsnForm + ? dsnFormId + : paramsFormId; $: submitting = (() => { - if (isMultiStepConnector) return multiStepSubmitting; - if (!formManager) return false; - if (onlyDsn || connectionTab === "dsn") { + if (useDsnForm) { return $dsnSubmitting; } else { return $paramsSubmitting; } })(); - $: primaryButtonLabel = isMultiStepConnector + $: primaryButtonLabel = isStepFlowConnector ? multiStepButtonLabel - : formManager - ? formManager.getPrimaryButtonLabel({ - isConnectorForm, - step: isSourceForm ? "source" : "connector", - submitting, - clickhouseConnectorType, - clickhouseSubmitting, - selectedAuthMethod: activeAuthMethod ?? undefined, - }) - : ""; + : formManager.getPrimaryButtonLabel({ + isConnectorForm, + step: stepState.step, + submitting, + clickhouseConnectorType, + clickhouseSubmitting: effectiveClickhouseSubmitting, + selectedAuthMethod: activeAuthMethod ?? undefined, + }); $: primaryLoadingCopy = (() => { if (connector.name === "clickhouse") return "Connecting..."; - if (isMultiStepConnector) return multiStepLoadingCopy; + if (isStepFlowConnector) return multiStepLoadingCopy; return activeAuthMethod === "public" ? "Continuing..." : "Testing connection..."; @@ -246,54 +301,31 @@ saveAnyway = false; } - $: isSubmitting = submitting; - - $: ctaDisabled = - connector.name === "clickhouse" - ? clickhouseSubmitting || clickhouseIsSubmitDisabled - : isMultiStepConnector - ? multiStepSubmitting || multiStepSubmitDisabled - : submitting || isSubmitDisabled; - $: ctaLoading = - connector.name === "clickhouse" - ? clickhouseSubmitting - : isMultiStepConnector - ? multiStepSubmitting - : submitting; - $: ctaLoadingCopy = isMultiStepConnector - ? multiStepLoadingCopy - : primaryLoadingCopy; - $: ctaLabel = isMultiStepConnector - ? multiStepButtonLabel - : primaryButtonLabel; - $: ctaFormId = + $: isSubmitting = connector.name === "clickhouse" - ? clickhouseFormId - : isMultiStepConnector - ? multiStepFormId - : formId; - - $: effectiveYaml = isMultiStepConnector ? multiStepYamlPreview : yamlPreview; - $: effectiveYamlTitle = isMultiStepConnector - ? multiStepYamlPreviewTitle - : isSourceForm - ? "Model preview" - : "Connector preview"; - - // Reset errors when form is modified (non-multi-step paths) + ? effectiveClickhouseSubmitting + : submitting; + + // Reset errors when form is modified $: (() => { - if (isMultiStepConnector || !formManager) return; - if (onlyDsn || connectionTab === "dsn") { + if (connector.name === "clickhouse") { + if ($paramsTainted) { + if (connectionTab === "dsn") { + dsnError = null; + } else { + paramsError = null; + } + } + } else if (useDsnForm) { if ($dsnTainted) dsnError = null; } else { if ($paramsTainted) paramsError = null; } })(); - // Clear errors when switching tabs (non-multi-step paths) + // Clear errors when switching tabs $: (() => { - if (isMultiStepConnector || !formManager) return; - if (hasDsnFormOption) { + if (usesLegacyTabs) { if (connectionTab === "dsn") { paramsError = null; paramsErrorDetails = undefined; @@ -310,31 +342,22 @@ return; } - // Multi-step connectors delegate to the container handler - if (isMultiStepConnector) { - await multiStepSaveAnywayHandler(); - return; - } - - if (!formManager) return; - // For other connectors, use manager helper saveAnyway = true; const values = connector.name === "clickhouse" - ? connectionTab === "dsn" - ? $clickhouseDsnForm - : $clickhouseParamsForm - : onlyDsn || connectionTab === "dsn" + ? $paramsForm + : useDsnForm ? $dsnForm : $paramsForm; if (connector.name === "clickhouse") { - clickhouseSubmitting = true; + clickhouseSaving = true; } const result = await formManager.saveConnectorAnyway({ queryClient, values, clickhouseConnectorType, + connectionTab, }); if (result.ok) { onClose(); @@ -347,7 +370,7 @@ paramsError = result.message; paramsErrorDetails = result.details; } - } else if (onlyDsn || connectionTab === "dsn") { + } else if (useDsnForm) { dsnError = result.message; dsnErrorDetails = result.details; } else { @@ -357,69 +380,53 @@ } saveAnyway = false; if (connector.name === "clickhouse") { - clickhouseSubmitting = false; + clickhouseSaving = false; } } - $: yamlPreview = isMultiStepConnector - ? multiStepYamlPreview - : formManager - ? formManager.computeYamlPreview({ - connectionTab, - onlyDsn, - filteredParamsProperties, - filteredDsnProperties, - stepState: undefined, - isMultiStepConnector, - isConnectorForm, - paramsFormValues: $paramsForm, - dsnFormValues: $dsnForm, - clickhouseConnectorType, - clickhouseParamsValues: $clickhouseParamsForm, - clickhouseDsnValues: $clickhouseDsnForm, - }) - : ""; + $: yamlPreview = formManager.computeYamlPreview({ + connectionTab, + onlyDsn, + filteredParamsProperties, + filteredDsnProperties, + stepState, + isMultiStepConnector: isStepFlowConnector, + isConnectorForm, + paramsFormValues: $paramsForm, + dsnFormValues: $dsnForm, + clickhouseConnectorType, + clickhouseParamsValues: $paramsForm, + clickhouseDsnValues: $paramsForm, + }); $: isClickhouse = connector.name === "clickhouse"; - $: shouldShowSaveAnywayButton = - isConnectorForm && - (clickhouseShowSaveAnyway || - (isMultiStepConnector ? multiStepShowSaveAnyway : showSaveAnyway)); - $: saveAnywayLoading = isMultiStepConnector - ? multiStepSaveAnywayLoading - : isClickhouse - ? clickhouseSubmitting && saveAnyway - : submitting && saveAnyway; - - if (formManager) { - const fm = formManager as AddDataFormManager; - handleOnUpdate = - fm.makeOnUpdate({ - onClose, - queryClient, - getConnectionTab: () => connectionTab, - getSelectedAuthMethod: () => activeAuthMethod || undefined, - setParamsError: (message: string | null, details?: string) => { - paramsError = message; - paramsErrorDetails = details; - }, - setDsnError: (message: string | null, details?: string) => { - dsnError = message; - dsnErrorDetails = details; - }, - setShowSaveAnyway: (value: boolean) => { - showSaveAnyway = value; - }, - }) ?? (async () => {}); - } else { - handleOnUpdate = async () => {}; - } + $: shouldShowSaveAnywayButton = isConnectorForm && showSaveAnyway; + $: saveAnywayLoading = isClickhouse + ? effectiveClickhouseSubmitting && saveAnyway + : submitting && saveAnyway; + + handleOnUpdate = formManager.makeOnUpdate({ + onClose, + queryClient, + getConnectionTab: () => connectionTab, + getSelectedAuthMethod: () => activeAuthMethod || undefined, + setParamsError: (message: string | null, details?: string) => { + paramsError = message; + paramsErrorDetails = details; + }, + setDsnError: (message: string | null, details?: string) => { + dsnError = message; + dsnErrorDetails = details; + }, + setShowSaveAnyway: (value: boolean) => { + showSaveAnyway = value; + }, + }); async function handleFileUpload(file: File): Promise { - return formManager ? formManager.handleFileUpload(file) : ""; + return formManager.handleFileUpload(file); } function onStringInputChange(event: Event) { - if (!formManager) return; formManager.onStringInputChange( event, $paramsTainted as Record | null | undefined, @@ -432,48 +439,25 @@
-
+
{#if connector.name === "clickhouse"} - { - clickhouseError = error; - clickhouseErrorDetails = details; - }} - bind:formId={clickhouseFormId} - bind:isSubmitting={clickhouseSubmitting} - bind:isSubmitDisabled={clickhouseIsSubmitDisabled} - bind:connectorType={clickhouseConnectorType} - bind:connectionTab - bind:paramsForm={clickhouseParamsForm} - bind:dsnForm={clickhouseDsnForm} - bind:showSaveAnyway={clickhouseShowSaveAnyway} - /> - {:else if isMultiStepConnector} - {#key connector.name} - + - {/key} - {:else if hasDsnFormOption} + + {:else if usesLegacyTabs} - @@ -500,28 +485,49 @@ enhance={dsnEnhance} onSubmit={dsnSubmit} > - - {:else if isConnectorForm && connector.configProperties?.some((property) => property.key === "dsn")} + {:else if isStepFlowConnector} + + {:else if hasSchema} - {:else} @@ -530,12 +536,13 @@ enhance={paramsEnhance} onSubmit={paramsSubmit} > - {/if} @@ -545,12 +552,8 @@
- formManager.handleBack(onBack)} type="secondary" + >Back
@@ -567,14 +570,19 @@ {/if}
@@ -588,27 +596,31 @@ {#if dsnError || paramsError || clickhouseError} {/if} - + {#if shouldShowSkipLink}
Already connected? diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 7bb829a05cc..85f27b425ac 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -1,22 +1,24 @@ import { superForm, defaults } from "sveltekit-superforms"; import type { SuperValidated } from "sveltekit-superforms"; +import * as yupLib from "yup"; import { - yup, + yup as yupAdapter, type Infer as YupInfer, type InferIn as YupInferIn, } from "sveltekit-superforms/adapters"; import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; import type { AddDataFormType } from "./types"; -import { getValidationSchemaForConnector, dsnSchema } from "./FormValidation"; -import { - getInitialFormValuesFromProperties, - inferSourceName, -} from "../sourceUtils"; +import { getValidationSchemaForConnector } from "./FormValidation"; +import { inferSourceName } from "../sourceUtils"; import { submitAddConnectorForm, submitAddSourceForm, } from "./submitAddDataForm"; -import { normalizeConnectorError } from "./utils"; +import { + normalizeConnectorError, + applyClickHouseCloudRequirements, + isEmpty, +} from "./utils"; import { FORM_HEIGHT_DEFAULT, FORM_HEIGHT_TALL, @@ -25,9 +27,8 @@ import { } from "./constants"; import { connectorStepStore, - resetConnectorStep, setConnectorConfig, - setAuthMethod, + setConnectorInstanceName, setStep, type ConnectorStepState, } from "./connectorStepStore"; @@ -36,10 +37,54 @@ import { compileConnectorYAML } from "../../connectors/code-utils"; import { compileSourceYAML, prepareSourceFormData } from "../sourceUtils"; import type { ConnectorDriverProperty } from "@rilldata/web-common/runtime-client"; import type { ClickHouseConnectorType } from "./constants"; -import { applyClickHouseCloudRequirements } from "./utils"; import type { ActionResult } from "@sveltejs/kit"; import { getConnectorSchema } from "./connector-schemas"; -import { findRadioEnumKey } from "../../templates/schema-utils"; +import { + filterSchemaValuesForSubmit, + findRadioEnumKey, + getRequiredFieldsForValues, + getSchemaFieldMetaList, + getSchemaInitialValues, + getSchemaSecretKeys, + getSchemaStringKeys, + isVisibleForValues, + type SchemaFieldMeta, +} from "../../templates/schema-utils"; + +const dsnSchema = yupLib.object({ + dsn: yupLib.string().required("DSN is required"), +}); + +export type ClickhouseUiState = { + properties: Array; + filteredProperties: Array; + dsnProperties: Array; + isSubmitDisabled: boolean; + formId: string; + submitting: boolean; + enforcedConnectionTab?: "parameters" | "dsn"; + shouldClearErrors?: boolean; +}; + +type SuperFormStore = { + update: ( + updater: (value: Record) => Record, + options?: any, + ) => void; +}; + +type ClickhouseStateArgs = { + connectorType: ClickHouseConnectorType; + connectionTab: "parameters" | "dsn"; + paramsFormValues: Record; + dsnFormValues: Record; + paramsErrors: Record; + dsnErrors: Record; + paramsForm: SuperFormStore; + dsnForm: SuperFormStore; + paramsSubmitting: boolean; + dsnSubmitting: boolean; +}; // Minimal onUpdate event type carrying Superforms's validated form type SuperFormUpdateEvent = { @@ -58,18 +103,17 @@ export class AddDataFormManager { dsnFormId: string; hasDsnFormOption: boolean; hasOnlyDsn: boolean; - properties: ConnectorDriverProperty[]; - filteredParamsProperties: ConnectorDriverProperty[]; - dsnProperties: ConnectorDriverProperty[]; - filteredDsnProperties: ConnectorDriverProperty[]; + properties: Array; + filteredParamsProperties: Array; + dsnProperties: Array; + filteredDsnProperties: Array; // superforms instances params: ReturnType; dsn: ReturnType; private connector: V1ConnectorDriver; private formType: AddDataFormType; - private initialParamsValues: Record; - private initialDsnValues: Record; + private clickhouseInitialValues: Record; // Centralized error normalization for this manager private normalizeError(e: unknown): { message: string; details?: string } { @@ -77,13 +121,15 @@ export class AddDataFormManager { } private getSelectedAuthMethod?: () => string | undefined; - private resetConnectorForms() { - if (this.params?.form) { - (this.params.form as any).update(() => ({}), { taint: false } as any); - } - if (this.dsn?.form) { - (this.dsn.form as any).update(() => ({}), { taint: false } as any); - } + // Keep only fields that belong to a given schema step. Prevents source-step + // values (e.g., URI/model) from leaking into connector state when we persist. + private filterValuesForStep( + values: Record, + step: "connector" | "source" | "explorer", + ): Record { + const schema = getConnectorSchema(this.connector.name ?? ""); + if (!schema?.properties) return values; + return filterSchemaValuesForSubmit(schema, values, { step }); } constructor(args: { @@ -114,55 +160,36 @@ export class AddDataFormManager { this.dsnFormId = `add-data-${connector.name}-dsn-form`; const isSourceForm = formType === "source"; - const isConnectorForm = formType === "connector"; + const schema = getConnectorSchema(connector.name ?? ""); + const schemaStep = isSourceForm ? "source" : "connector"; + const schemaFields = schema + ? getSchemaFieldMetaList(schema, { step: schemaStep }) + : []; // Base properties - this.properties = - (isSourceForm - ? connector.sourceProperties - : connector.configProperties?.filter((p) => p.key !== "dsn")) ?? []; + this.properties = schemaFields; // Filter properties based on connector type - this.filteredParamsProperties = (() => { - if (connector.name === "duckdb") { - return this.properties.filter( - (p) => p.key !== "attach" && p.key !== "mode", - ); - } - return this.properties.filter((p) => !p.noPrompt); - })(); + this.filteredParamsProperties = this.properties; // DSN properties - this.dsnProperties = - connector.configProperties?.filter((p) => p.key === "dsn") ?? []; + this.dsnProperties = schemaFields.filter((field) => field.key === "dsn"); this.filteredDsnProperties = this.dsnProperties; // DSN flags - this.hasDsnFormOption = !!( - isConnectorForm && - connector.configProperties?.some((p) => p.key === "dsn") && - connector.configProperties?.some((p) => p.key !== "dsn") - ); - this.hasOnlyDsn = !!( - isConnectorForm && - connector.configProperties?.some((p) => p.key === "dsn") && - !connector.configProperties?.some((p) => p.key !== "dsn") - ); + this.hasDsnFormOption = false; + this.hasOnlyDsn = false; // Superforms: params const paramsAdapter = getValidationSchemaForConnector( connector.name as string, formType, - { - isMultiStepConnector: this.isMultiStepConnector, - }, ); type ParamsOut = Record; type ParamsIn = Record; - const initialFormValues = getInitialFormValuesFromProperties( - this.properties, - ); - this.initialParamsValues = initialFormValues; + const initialFormValues = schema + ? getSchemaInitialValues(schema, { step: schemaStep }) + : {}; const paramsDefaults = defaults( initialFormValues as Partial, paramsAdapter, @@ -176,18 +203,22 @@ export class AddDataFormManager { }); // Superforms: dsn - const dsnAdapter = yup(dsnSchema); + const dsnAdapter = yupAdapter(dsnSchema); type DsnOut = YupInfer; type DsnIn = YupInferIn; - const initialDsnValues = defaults(dsnAdapter); - this.initialDsnValues = initialDsnValues; - this.dsn = superForm(initialDsnValues, { + this.dsn = superForm(defaults(dsnAdapter), { SPA: true, validators: dsnAdapter, onUpdate: onDsnUpdate, resetForm: false, validationMethod: "onsubmit", }); + + // ClickHouse-specific defaults + this.clickhouseInitialValues = + connector.name === "clickhouse" && schema + ? getSchemaInitialValues(schema, { step: "connector" }) + : {}; } get isSourceForm(): boolean { @@ -202,6 +233,15 @@ export class AddDataFormManager { return MULTI_STEP_CONNECTORS.includes(this.connector.name ?? ""); } + get isExplorerConnector(): boolean { + return Boolean( + (this.connector.implementsOlap || + this.connector.implementsSqlStore || + this.connector.implementsWarehouse) && + this.connector.name !== "clickhouse", + ); + } + /** * Determines whether the "Save Anyway" button should be shown for the current submission. */ @@ -220,14 +260,12 @@ export class AddDataFormManager { // Only show for connector forms (not sources) if (!isConnectorForm) return false; - // ClickHouse has its own error handling - if (this.connector.name === "clickhouse") return false; - // Need a submission result to show the button if (!event?.result) return false; - // Multi-step connectors: don't show on source step (final step) - if (stepState?.step === "source") return false; + // Multi-step connectors: don't show on source/explorer step (final step) + if (stepState?.step === "source" || stepState?.step === "explorer") + return false; // Public auth bypasses connection test, so no "Save Anyway" needed if (stepState?.step === "connector" && selectedAuthMethod === "public") @@ -250,18 +288,15 @@ export class AddDataFormManager { const stepState = get(connectorStepStore) as ConnectorStepState; if (!this.isMultiStepConnector || stepState.step !== "connector") return; setConnectorConfig({}); - setAuthMethod(null); - this.resetConnectorForms(); + setConnectorInstanceName(null); setStep("source"); } handleBack(onBack: () => void): void { const stepState = get(connectorStepStore) as ConnectorStepState; if (this.isMultiStepConnector && stepState.step === "source") { - // Clear any connector form state when navigating back from the model step - setConnectorConfig(null); - setAuthMethod(null); - this.resetConnectorForms(); + setStep("connector"); + } else if (this.isExplorerConnector && stepState.step === "explorer") { setStep("connector"); } else { onBack(); @@ -285,6 +320,8 @@ export class AddDataFormManager { selectedAuthMethod, } = args; const isClickhouse = this.connector.name === "clickhouse"; + const isStepFlowConnector = + this.isMultiStepConnector || this.isExplorerConnector; if (isClickhouse) { if (clickhouseConnectorType === "rill-managed") { @@ -296,7 +333,7 @@ export class AddDataFormManager { } if (isConnectorForm) { - if (this.isMultiStepConnector && step === "connector") { + if (isStepFlowConnector && step === "connector") { if (selectedAuthMethod === "public") { return submitting ? BUTTON_LABELS.public.submitting @@ -306,7 +343,7 @@ export class AddDataFormManager { ? BUTTON_LABELS.connector.submitting : BUTTON_LABELS.connector.idle; } - if (this.isMultiStepConnector && step === "source") { + if (isStepFlowConnector && (step === "source" || step === "explorer")) { return submitting ? BUTTON_LABELS.source.submitting : BUTTON_LABELS.source.idle; @@ -319,6 +356,106 @@ export class AddDataFormManager { return "Test and Add data"; } + computeClickhouseState(args: ClickhouseStateArgs): ClickhouseUiState | null { + if (this.connector.name !== "clickhouse") return null; + const { + connectorType, + connectionTab, + paramsFormValues, + dsnFormValues, + paramsErrors, + dsnErrors, + paramsForm, + paramsSubmitting, + dsnSubmitting, + } = args; + const schema = getConnectorSchema(this.connector.name ?? ""); + + // Keep connector_type in sync on the params form + if (paramsFormValues?.connector_type !== connectorType) { + paramsForm.update( + ($form: any) => ({ + ...$form, + connector_type: connectorType, + }), + { taint: false } as any, + ); + } + + const enforcedConnectionTab = + connectorType === "rill-managed" ? ("parameters" as const) : undefined; + const activeConnectionTab = enforcedConnectionTab ?? connectionTab; + + const requiredFields = schema + ? getRequiredFieldsForValues(schema, paramsFormValues ?? {}, "connector") + : new Set(); + const isSubmitDisabled = Array.from(requiredFields).some((key) => { + if (schema && !isVisibleForValues(schema, key, paramsFormValues ?? {})) { + return false; + } + const errorsForField = + activeConnectionTab === "dsn" + ? (dsnErrors?.[key] as any) + : (paramsErrors?.[key] as any); + const value = + activeConnectionTab === "dsn" + ? dsnFormValues?.[key] + : paramsFormValues?.[key]; + return isEmpty(value) || Boolean(errorsForField?.length); + }); + + const submitting = + activeConnectionTab === "dsn" ? dsnSubmitting : paramsSubmitting; + const formId = + activeConnectionTab === "dsn" ? this.dsnFormId : this.paramsFormId; + + return { + properties: this.properties, + filteredProperties: this.filteredParamsProperties, + dsnProperties: this.dsnProperties, + isSubmitDisabled, + formId, + submitting, + enforcedConnectionTab, + shouldClearErrors: connectorType === "rill-managed", + }; + } + + getClickhouseDefaults( + connectorType: ClickHouseConnectorType, + ): Record | null { + if (this.connector.name !== "clickhouse") return null; + const baseDefaults = { ...this.clickhouseInitialValues }; + delete (baseDefaults as Record).connector_type; + + if (connectorType === "clickhouse-cloud") { + return { + ...baseDefaults, + managed: false, + port: "8443", + ssl: true, + connector_type: "clickhouse-cloud", + connection_mode: "parameters", + }; + } + + if (connectorType === "rill-managed") { + return { + ...baseDefaults, + managed: true, + connector_type: "rill-managed", + connection_mode: "parameters", + }; + } + + return { + ...baseDefaults, + managed: false, + connector_type: "self-hosted", + connection_mode: "parameters", + }; + } + makeOnUpdate(args: { onClose: () => void; queryClient: any; @@ -341,6 +478,8 @@ export class AddDataFormManager { const isMultiStepConnector = MULTI_STEP_CONNECTORS.includes( connector.name ?? "", ); + const isExplorerConnector = this.isExplorerConnector; + const isStepFlowConnector = isMultiStepConnector || isExplorerConnector; const isConnectorForm = this.formType === "connector"; return async (event: { @@ -353,7 +492,25 @@ export class AddDataFormManager { cancel?: () => void; }) => { const values = event.form.data; + const connectionTab = getConnectionTab(); const schema = getConnectorSchema(this.connector.name ?? ""); + const stepState = get(connectorStepStore) as ConnectorStepState; + const stepForFilter = + isStepFlowConnector && + (stepState.step === "source" || stepState.step === "explorer") + ? stepState.step + : this.formType === "source" + ? "source" + : "connector"; + const filteredValues = schema + ? filterSchemaValuesForSubmit(schema, values, { + step: stepForFilter, + }) + : values; + const submitValues = + this.connector.name === "clickhouse" + ? this.filterClickhouseValues(filteredValues, connectionTab) + : filteredValues; const authKey = schema ? findRadioEnumKey(schema) : null; const selectedAuthMethod = (authKey && values && values[authKey] != null @@ -361,25 +518,26 @@ export class AddDataFormManager { : undefined) || getSelectedAuthMethod?.() || ""; - const stepState = get(connectorStepStore) as ConnectorStepState; - // Fast-path: public auth skips validation/test and goes straight to source step. if ( isMultiStepConnector && stepState.step === "connector" && selectedAuthMethod === "public" ) { - setConnectorConfig({}); - setAuthMethod(null); + const connectorValues = this.filterValuesForStep(values, "connector"); + setConnectorConfig(connectorValues); setStep("source"); return; } - if (isMultiStepConnector && stepState.step === "source") { + if ( + isStepFlowConnector && + (stepState.step === "source" || stepState.step === "explorer") + ) { const sourceValidator = getValidationSchemaForConnector( connector.name as string, "source", - { isMultiStepConnector: true }, + stepState.step, ); const result = await sourceValidator.validate(values); if (!result.success) { @@ -412,38 +570,67 @@ export class AddDataFormManager { } try { - if (isMultiStepConnector && stepState.step === "source") { - await submitAddSourceForm(queryClient, connector, values); - resetConnectorStep(); - this.resetConnectorForms(); + if ( + isStepFlowConnector && + (stepState.step === "source" || stepState.step === "explorer") + ) { + const connectorInstanceName = + stepState.connectorInstanceName ?? undefined; + await submitAddSourceForm( + queryClient, + connector, + submitValues, + connectorInstanceName, + ); onClose(); - } else if (isMultiStepConnector && stepState.step === "connector") { + } else if (isStepFlowConnector && stepState.step === "connector") { // For public auth, skip Test & Connect and go straight to the next step. if (selectedAuthMethod === "public") { - setConnectorConfig({}); - setAuthMethod(null); - this.resetConnectorForms(); - setStep("source"); + const connectorValues = this.filterValuesForStep( + values, + "connector", + ); + setConnectorConfig(connectorValues); + setConnectorInstanceName(null); + if (isMultiStepConnector) { + setStep("source"); + } else if (isExplorerConnector) { + setStep("explorer"); + } return; } - await submitAddConnectorForm(queryClient, connector, values, false); - setConnectorConfig({}); - setAuthMethod(null); - this.resetConnectorForms(); - setStep("source"); + const connectorInstanceName = await submitAddConnectorForm( + queryClient, + connector, + submitValues, + false, + ); + const connectorValues = this.filterValuesForStep( + submitValues, + "connector", + ); + setConnectorConfig(connectorValues); + setConnectorInstanceName(connectorInstanceName); + if (isMultiStepConnector) { + setStep("source"); + } else if (isExplorerConnector) { + setStep("explorer"); + } return; } else if (this.formType === "source") { - await submitAddSourceForm(queryClient, connector, values); - resetConnectorStep(); - this.resetConnectorForms(); + await submitAddSourceForm(queryClient, connector, submitValues); onClose(); } else { - await submitAddConnectorForm(queryClient, connector, values, false); + await submitAddConnectorForm( + queryClient, + connector, + submitValues, + false, + ); onClose(); } } catch (e) { const { message, details } = this.normalizeError(e); - const connectionTab = getConnectionTab(); if (isConnectorForm && (this.hasOnlyDsn || connectionTab === "dsn")) { setDsnError(message, details); } else { @@ -461,6 +648,22 @@ export class AddDataFormManager { ) => { const target = event.target as HTMLInputElement; const { name, value } = target; + const key = name || target.id; + + // Clear stale field-level errors as soon as the user edits the input. + const clearFieldError = (store: any) => { + if (!store?.update || !key) return; + store.update(($errors: Record) => { + if (!$errors || !Object.prototype.hasOwnProperty.call($errors, key)) { + return $errors; + } + const next = { ...$errors }; + delete next[key]; + return next; + }); + }; + clearFieldError(this.params.errors); + clearFieldError(this.dsn.errors); if (name === "path") { const nameTainted = taintedFields && typeof taintedFields === "object" @@ -512,8 +715,8 @@ export class AddDataFormManager { computeYamlPreview(ctx: { connectionTab: "parameters" | "dsn"; onlyDsn: boolean; - filteredParamsProperties: ConnectorDriverProperty[]; - filteredDsnProperties: ConnectorDriverProperty[]; + filteredParamsProperties: Array; + filteredDsnProperties: Array; stepState: ConnectorStepState | undefined; isMultiStepConnector: boolean; isConnectorForm: boolean; @@ -539,22 +742,36 @@ export class AddDataFormManager { clickhouseDsnValues, } = ctx; - const connectorPropertiesForPreview = - isMultiStepConnector && stepState?.step === "connector" - ? (connector.configProperties ?? []) - : filteredParamsProperties; + const schema = getConnectorSchema(connector.name ?? ""); + const schemaConnectorFields = schema + ? getSchemaFieldMetaList(schema, { step: "connector" }) + : null; + const schemaConnectorSecretKeys = schema + ? getSchemaSecretKeys(schema, { step: "connector" }) + : undefined; + const schemaConnectorStringKeys = schema + ? getSchemaStringKeys(schema, { step: "connector" }) + : undefined; + + const connectorPropertiesForPreview = schemaConnectorFields ?? []; const getConnectorYamlPreview = (values: Record) => { + const filteredValues = schema + ? filterSchemaValuesForSubmit(schema, values, { step: "connector" }) + : values; const orderedProperties = onlyDsn || connectionTab === "dsn" ? filteredDsnProperties : connectorPropertiesForPreview; - return compileConnectorYAML(connector, values, { + return compileConnectorYAML(connector, filteredValues, { fieldFilter: (property) => { if (onlyDsn || connectionTab === "dsn") return true; - return !property.noPrompt; + if ("internal" in property && property.internal) return false; + return !("noPrompt" in property && property.noPrompt); }, orderedProperties, + secretKeys: schemaConnectorSecretKeys, + stringKeys: schemaConnectorStringKeys, }); }; @@ -562,9 +779,15 @@ export class AddDataFormManager { values: Record, chType: ClickHouseConnectorType | undefined, ) => { + const filteredValues = schema + ? filterSchemaValuesForSubmit(schema, values, { step: "connector" }) + : values; // Convert to managed boolean and apply CH Cloud requirements for preview const managed = chType === "rill-managed"; - const previewValues = { ...values, managed } as Record; + const previewValues = { + ...filteredValues, + managed, + } as Record; const finalValues = applyClickHouseCloudRequirements( connector.name, chType as ClickHouseConnectorType, @@ -573,12 +796,15 @@ export class AddDataFormManager { return compileConnectorYAML(connector, finalValues, { fieldFilter: (property) => { if (onlyDsn || connectionTab === "dsn") return true; - return !property.noPrompt; + if ("internal" in property && property.internal) return false; + return !("noPrompt" in property && property.noPrompt); }, orderedProperties: connectionTab === "dsn" ? filteredDsnProperties : filteredParamsProperties, + secretKeys: schemaConnectorSecretKeys, + stringKeys: schemaConnectorStringKeys, }); }; @@ -587,7 +813,11 @@ export class AddDataFormManager { let filteredValues = values; if (isMultiStepConnector && stepState?.step === "source") { const connectorPropertyKeys = new Set( - connector.configProperties?.map((p) => p.key).filter(Boolean) || [], + schema + ? getSchemaFieldMetaList(schema, { step: "connector" }) + .filter((field) => !field.internal) + .map((field) => field.key) + : [], ); filteredValues = Object.fromEntries( Object.entries(values).filter( @@ -599,10 +829,25 @@ export class AddDataFormManager { const [rewrittenConnector, rewrittenFormValues] = prepareSourceFormData( connector, filteredValues, + { + connectorInstanceName: stepState?.connectorInstanceName || undefined, + }, ); + const isExplorerStep = stepState?.step === "explorer"; const isRewrittenToDuckDb = rewrittenConnector.name === "duckdb"; - if (isRewrittenToDuckDb) { - return compileSourceYAML(rewrittenConnector, rewrittenFormValues); + const rewrittenSchema = getConnectorSchema(rewrittenConnector.name ?? ""); + const sourceStep = isExplorerStep ? "explorer" : "source"; + const rewrittenSecretKeys = rewrittenSchema + ? getSchemaSecretKeys(rewrittenSchema, { step: sourceStep }) + : undefined; + const rewrittenStringKeys = rewrittenSchema + ? getSchemaStringKeys(rewrittenSchema, { step: sourceStep }) + : undefined; + if (isRewrittenToDuckDb || isExplorerStep) { + return compileSourceYAML(rewrittenConnector, rewrittenFormValues, { + secretKeys: rewrittenSecretKeys, + stringKeys: rewrittenStringKeys, + }); } return getConnectorYamlPreview(rewrittenFormValues); }; @@ -642,13 +887,21 @@ export class AddDataFormManager { queryClient: any; values: Record; clickhouseConnectorType?: ClickHouseConnectorType; + connectionTab?: "parameters" | "dsn"; }): Promise<{ ok: true } | { ok: false; message: string; details?: string }> { - const { queryClient, values, clickhouseConnectorType } = args; + const { queryClient, values, clickhouseConnectorType, connectionTab } = + args; + const tab = connectionTab ?? "parameters"; + const schema = getConnectorSchema(this.connector.name ?? ""); + const prunedValues = schema + ? filterSchemaValuesForSubmit(schema, values, { step: "connector" }) + : values; + const filteredValues = this.filterClickhouseValues(prunedValues, tab); const processedValues = applyClickHouseCloudRequirements( this.connector.name, (clickhouseConnectorType as ClickHouseConnectorType) || ("self-hosted" as ClickHouseConnectorType), - values, + filteredValues, ); try { await submitAddConnectorForm( @@ -663,4 +916,22 @@ export class AddDataFormManager { return { ok: false, message, details } as const; } } + + private filterClickhouseValues( + values: Record, + connectionTab: "parameters" | "dsn", + ): Record { + if (this.connector.name !== "clickhouse") return values; + if (connectionTab !== "dsn") { + if (!("dsn" in values)) return values; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { dsn: _unused, ...rest } = values; + return rest; + } + + const allowed = new Set(["dsn", "managed"]); + return Object.fromEntries( + Object.entries(values).filter(([key]) => allowed.has(key)), + ); + } } diff --git a/web-common/src/features/sources/modal/ConnectorForm.svelte b/web-common/src/features/sources/modal/ConnectorForm.svelte deleted file mode 100644 index 513b389277d..00000000000 --- a/web-common/src/features/sources/modal/ConnectorForm.svelte +++ /dev/null @@ -1,277 +0,0 @@ - - - diff --git a/web-common/src/features/sources/modal/FormRenderer.svelte b/web-common/src/features/sources/modal/FormRenderer.svelte deleted file mode 100644 index dc33afc5fff..00000000000 --- a/web-common/src/features/sources/modal/FormRenderer.svelte +++ /dev/null @@ -1,63 +0,0 @@ - - -{#each properties as property (property.key)} - {@const propertyKey = property.key ?? ""} - {@const label = - property.displayName + (property.required ? "" : " (optional)")} -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_FILE} - - {/if} -
-{/each} diff --git a/web-common/src/features/sources/modal/FormValidation.test.ts b/web-common/src/features/sources/modal/FormValidation.test.ts index 9df8111c168..722052755c4 100644 --- a/web-common/src/features/sources/modal/FormValidation.test.ts +++ b/web-common/src/features/sources/modal/FormValidation.test.ts @@ -4,9 +4,7 @@ import { getValidationSchemaForConnector } from "./FormValidation"; describe("getValidationSchemaForConnector (multi-step auth)", () => { it("enforces required fields for access key auth", async () => { - const schema = getValidationSchemaForConnector("s3", "connector", { - isMultiStepConnector: true, - }); + const schema = getValidationSchemaForConnector("s3", "connector"); const result = await schema.validate({}); expect(result.success).toBe(false); @@ -20,18 +18,14 @@ describe("getValidationSchemaForConnector (multi-step auth)", () => { }); it("allows public auth without credentials", async () => { - const schema = getValidationSchemaForConnector("s3", "connector", { - isMultiStepConnector: true, - }); + const schema = getValidationSchemaForConnector("s3", "connector"); const result = await schema.validate({ auth_method: "public" }); expect(result.success).toBe(true); }); it("requires source fields from JSON schema for multi-step connectors", async () => { - const schema = getValidationSchemaForConnector("s3", "source", { - isMultiStepConnector: true, - }); + const schema = getValidationSchemaForConnector("s3", "source"); const result = await schema.validate({}); expect(result.success).toBe(false); @@ -45,9 +39,7 @@ describe("getValidationSchemaForConnector (multi-step auth)", () => { }); it("rejects invalid s3 path on source step", async () => { - const schema = getValidationSchemaForConnector("s3", "source", { - isMultiStepConnector: true, - }); + const schema = getValidationSchemaForConnector("s3", "source"); const result = await schema.validate({ path: "s3:/bucket", @@ -61,9 +53,7 @@ describe("getValidationSchemaForConnector (multi-step auth)", () => { }); it("accepts valid s3 path on source step", async () => { - const schema = getValidationSchemaForConnector("s3", "source", { - isMultiStepConnector: true, - }); + const schema = getValidationSchemaForConnector("s3", "source"); const result = await schema.validate({ path: "s3://bucket/prefix", diff --git a/web-common/src/features/sources/modal/FormValidation.ts b/web-common/src/features/sources/modal/FormValidation.ts index 568d6862f7d..9d0fbe477ec 100644 --- a/web-common/src/features/sources/modal/FormValidation.ts +++ b/web-common/src/features/sources/modal/FormValidation.ts @@ -1,28 +1,17 @@ import type { ValidationAdapter } from "sveltekit-superforms/adapters"; -import { yup as yupAdapter } from "sveltekit-superforms/adapters"; import { createSchemasafeValidator } from "./jsonSchemaValidator"; import { getConnectorSchema } from "./connector-schemas"; -import { dsnSchema, getYupSchema } from "./yupSchemas"; import type { AddDataFormType } from "./types"; -export { dsnSchema }; - export function getValidationSchemaForConnector( name: string, formType: AddDataFormType, - opts?: { - isMultiStepConnector?: boolean; - }, + stepOverride?: "connector" | "source" | "explorer", ): ValidationAdapter> { - const { isMultiStepConnector } = opts || {}; const jsonSchema = getConnectorSchema(name); + const step = stepOverride ?? (formType === "source" ? "source" : "connector"); - if (isMultiStepConnector && jsonSchema) { - const step = formType === "source" ? "source" : "connector"; - return createSchemasafeValidator(jsonSchema, step); - } - - const fallbackYupSchema = getYupSchema[name as keyof typeof getYupSchema]; - return yupAdapter(fallbackYupSchema); + if (jsonSchema) return createSchemasafeValidator(jsonSchema, step); + throw new Error(`No validation schema found for connector: ${name}`); } diff --git a/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte b/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte new file mode 100644 index 00000000000..7fb80c5e2aa --- /dev/null +++ b/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte @@ -0,0 +1,209 @@ + + + + + diff --git a/web-common/src/features/sources/modal/connector-schemas.ts b/web-common/src/features/sources/modal/connector-schemas.ts index 0b7cff5363b..0e2346fbbd5 100644 --- a/web-common/src/features/sources/modal/connector-schemas.ts +++ b/web-common/src/features/sources/modal/connector-schemas.ts @@ -1,9 +1,39 @@ import type { MultiStepFormSchema } from "../../templates/schemas/types"; +import { athenaSchema } from "../../templates/schemas/athena"; import { azureSchema } from "../../templates/schemas/azure"; +import { bigquerySchema } from "../../templates/schemas/bigquery"; +import { clickhouseSchema } from "../../templates/schemas/clickhouse"; import { gcsSchema } from "../../templates/schemas/gcs"; +import { mysqlSchema } from "../../templates/schemas/mysql"; +import { postgresSchema } from "../../templates/schemas/postgres"; +import { redshiftSchema } from "../../templates/schemas/redshift"; +import { salesforceSchema } from "../../templates/schemas/salesforce"; +import { snowflakeSchema } from "../../templates/schemas/snowflake"; +import { sqliteSchema } from "../../templates/schemas/sqlite"; +import { localFileSchema } from "../../templates/schemas/local_file"; +import { duckdbSchema } from "../../templates/schemas/duckdb"; +import { httpsSchema } from "../../templates/schemas/https"; +import { motherduckSchema } from "../../templates/schemas/motherduck"; +import { druidSchema } from "../../templates/schemas/druid"; +import { pinotSchema } from "../../templates/schemas/pinot"; import { s3Schema } from "../../templates/schemas/s3"; export const multiStepFormSchemas: Record = { + athena: athenaSchema, + bigquery: bigquerySchema, + clickhouse: clickhouseSchema, + mysql: mysqlSchema, + postgres: postgresSchema, + redshift: redshiftSchema, + salesforce: salesforceSchema, + snowflake: snowflakeSchema, + sqlite: sqliteSchema, + motherduck: motherduckSchema, + duckdb: duckdbSchema, + druid: druidSchema, + pinot: pinotSchema, + local_file: localFileSchema, + https: httpsSchema, s3: s3Schema, gcs: gcsSchema, azure: azureSchema, diff --git a/web-common/src/features/sources/modal/connectorStepStore.ts b/web-common/src/features/sources/modal/connectorStepStore.ts index 8a20232e9be..22721d5738d 100644 --- a/web-common/src/features/sources/modal/connectorStepStore.ts +++ b/web-common/src/features/sources/modal/connectorStepStore.ts @@ -1,16 +1,18 @@ import { writable } from "svelte/store"; -export type ConnectorStep = "connector" | "source"; +export type ConnectorStep = "connector" | "source" | "explorer"; export type ConnectorStepState = { step: ConnectorStep; connectorConfig: Record | null; + connectorInstanceName: string | null; selectedAuthMethod: string | null; }; export const connectorStepStore = writable({ step: "connector", connectorConfig: null, + connectorInstanceName: null, selectedAuthMethod: null, }); @@ -18,10 +20,17 @@ export function setStep(step: ConnectorStep) { connectorStepStore.update((state) => ({ ...state, step })); } -export function setConnectorConfig(config: Record | null) { +export function setConnectorConfig(config: Record) { connectorStepStore.update((state) => ({ ...state, connectorConfig: config })); } +export function setConnectorInstanceName(name: string | null) { + connectorStepStore.update((state) => ({ + ...state, + connectorInstanceName: name, + })); +} + export function setAuthMethod(method: string | null) { connectorStepStore.update((state) => ({ ...state, @@ -33,6 +42,7 @@ export function resetConnectorStep() { connectorStepStore.set({ step: "connector", connectorConfig: null, + connectorInstanceName: null, selectedAuthMethod: null, }); } diff --git a/web-common/src/features/sources/modal/properties-to-schema.ts b/web-common/src/features/sources/modal/properties-to-schema.ts new file mode 100644 index 00000000000..1f99fbd417d --- /dev/null +++ b/web-common/src/features/sources/modal/properties-to-schema.ts @@ -0,0 +1,108 @@ +import type { ConnectorDriverProperty } from "@rilldata/web-common/runtime-client"; +import { ConnectorDriverPropertyType } from "@rilldata/web-common/runtime-client"; +import type { + MultiStepFormSchema, + JSONSchemaField, +} from "../../templates/schemas/types"; + +/** + * Converts ConnectorDriverProperty[] to JSON Schema format. + * + * This is a translation layer at the API boundary - the ONLY place where + * ConnectorDriverPropertyType should be used. Once converted, JSON Schema + * types ("string" | "number" | "boolean" | "object") are the single source of truth. + * + * TODO: Once backend sends JSON Schema directly, this function can be removed. + */ +export function propertiesToSchema( + properties: ConnectorDriverProperty[], + step?: "connector" | "source", +): MultiStepFormSchema { + const schemaProperties: Record = {}; + const required: string[] = []; + + for (const prop of properties) { + if (!prop.key) continue; + + // Convert backend enum to JSON Schema type (the single source of truth) + const field: JSONSchemaField = { + type: convertBackendTypeToJsonSchemaType(prop.type), + title: prop.displayName || prop.key, + description: prop.description, + "x-placeholder": prop.placeholder, + "x-hint": prop.hint, + "x-secret": prop.secret || false, + "x-docs-url": prop.docsUrl, + }; + + // Convert default value based on JSON Schema type from the field itself + if ( + prop.default !== undefined && + prop.default !== null && + prop.default !== "" + ) { + if (field.type === "number") { + const num = Number(prop.default); + if (!isNaN(num)) { + field.default = num; + } + } else if (field.type === "boolean") { + field.default = prop.default === "true"; + } else { + field.default = prop.default; + } + } + + // Map backend-specific types to JSON Schema extensions + // (These are the only places we need to check the backend enum) + if (prop.type === ConnectorDriverPropertyType.TYPE_FILE) { + field["x-display"] = "file"; + field.format = "file"; + field["x-accept"] = ".json"; + } else if (prop.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL) { + field["x-informational"] = true; + } + + // Set step if provided + if (step) { + field["x-step"] = step; + } + + schemaProperties[prop.key] = field; + + // Add to required array if the property is required + if (prop.required) { + required.push(prop.key); + } + } + + return { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: schemaProperties, + required, + }; +} + +/** + * Translation layer: converts backend enum to JSON Schema type. + * This is the ONLY place ConnectorDriverPropertyType should be referenced. + * + * Once backend sends JSON Schema directly, this function can be removed. + */ +function convertBackendTypeToJsonSchemaType( + type?: ConnectorDriverPropertyType, +): "string" | "number" | "boolean" | "object" { + switch (type) { + case ConnectorDriverPropertyType.TYPE_NUMBER: + return "number"; + case ConnectorDriverPropertyType.TYPE_BOOLEAN: + return "boolean"; + case ConnectorDriverPropertyType.TYPE_STRING: + case ConnectorDriverPropertyType.TYPE_FILE: + case ConnectorDriverPropertyType.TYPE_INFORMATIONAL: + case ConnectorDriverPropertyType.TYPE_UNSPECIFIED: + default: + return "string"; + } +} diff --git a/web-common/src/features/sources/modal/submitAddDataForm.ts b/web-common/src/features/sources/modal/submitAddDataForm.ts index d9bdbd6f332..c501295271e 100644 --- a/web-common/src/features/sources/modal/submitAddDataForm.ts +++ b/web-common/src/features/sources/modal/submitAddDataForm.ts @@ -35,6 +35,12 @@ import { EMPTY_PROJECT_TITLE } from "../../welcome/constants"; import { isProjectInitialized } from "../../welcome/is-project-initialized"; import { compileSourceYAML, prepareSourceFormData } from "../sourceUtils"; import { OLAP_ENGINES } from "./constants"; +import { getConnectorSchema } from "./connector-schemas"; +import { + getSchemaFieldMetaList, + getSchemaSecretKeys, + getSchemaStringKeys, +} from "../../templates/schema-utils"; interface AddDataFormValues { // name: string; // Commenting out until we add user-provided names for Connectors @@ -48,9 +54,8 @@ const savedAnywayPaths = new Set(); const connectorSubmissions = new Map< string, { - promise: Promise; + promise: Promise; connectorName: string; - completed: boolean; } >(); @@ -162,6 +167,16 @@ async function saveConnectorAnyway( instanceId?: string, ): Promise { const resolvedInstanceId = instanceId ?? get(runtime).instanceId; + const schema = getConnectorSchema(connector.name ?? ""); + const schemaFields = schema + ? getSchemaFieldMetaList(schema, { step: "connector" }) + : []; + const schemaSecretKeys = schema + ? getSchemaSecretKeys(schema, { step: "connector" }) + : []; + const schemaStringKeys = schema + ? getSchemaStringKeys(schema, { step: "connector" }) + : []; // Create connector file const newConnectorFilePath = getFileAPIPathFromNameAndType( @@ -179,6 +194,7 @@ async function saveConnectorAnyway( formValues, "connector", newConnectorName, + { secretKeys: schemaSecretKeys }, ); await runtimeServicePutFile(resolvedInstanceId, { @@ -193,6 +209,12 @@ async function saveConnectorAnyway( path: newConnectorFilePath, blob: compileConnectorYAML(connector, formValues, { connectorInstanceName: newConnectorName, + orderedProperties: schemaFields, + secretKeys: schemaSecretKeys, + stringKeys: schemaStringKeys, + fieldFilter: schemaFields + ? (property) => !("internal" in property && property.internal) + : undefined, }), create: true, createOnly: false, @@ -215,9 +237,19 @@ export async function submitAddConnectorForm( connector: V1ConnectorDriver, formValues: AddDataFormValues, saveAnyway: boolean = false, -): Promise { +): Promise { const instanceId = get(runtime).instanceId; await beforeSubmitForm(instanceId, connector); + const schema = getConnectorSchema(connector.name ?? ""); + const schemaFields = schema + ? getSchemaFieldMetaList(schema, { step: "connector" }) + : []; + const schemaSecretKeys = schema + ? getSchemaSecretKeys(schema, { step: "connector" }) + : []; + const schemaStringKeys = schema + ? getSchemaStringKeys(schema, { step: "connector" }) + : []; // Create a unique key for this connector submission const uniqueConnectorSubmissionKey = `${instanceId}:${connector.name}`; @@ -236,7 +268,6 @@ export async function submitAddConnectorForm( if (saveAnyway) { // If Save Anyway is clicked while Test and Connect is running, // proceed immediately without waiting for the ongoing operation - // Clean up the existing submission connectorSubmissions.delete(uniqueConnectorSubmissionKey); // Use the same connector name from the ongoing operation @@ -250,13 +281,12 @@ export async function submitAddConnectorForm( newConnectorName, instanceId, ); - return; - } else if (!existingSubmission.completed) { - // If Test and Connect is clicked while another operation is running, - // wait for it to complete - await existingSubmission.promise; - return; + return newConnectorName; } + + // If Test and Connect is clicked while another operation is running, + // wait for it to complete + return existingSubmission.promise; } // Create abort controller for this submission @@ -288,6 +318,7 @@ export async function submitAddConnectorForm( formValues, "connector", newConnectorName, + { secretKeys: schemaSecretKeys }, ); if (saveAnyway) { @@ -299,7 +330,7 @@ export async function submitAddConnectorForm( newConnectorName, instanceId, ); - return; + return newConnectorName; } /** @@ -322,6 +353,12 @@ export async function submitAddConnectorForm( path: newConnectorFilePath, blob: compileConnectorYAML(connector, formValues, { connectorInstanceName: newConnectorName, + orderedProperties: schemaFields, + secretKeys: schemaSecretKeys, + stringKeys: schemaStringKeys, + fieldFilter: schemaFields + ? (property) => !("internal" in property && property.internal) + : undefined, }), create: true, createOnly: false, @@ -359,11 +396,12 @@ export async function submitAddConnectorForm( // Go to the new connector file await goto(`/files/${newConnectorFilePath}`); + return newConnectorName; } catch (error) { // If the operation was aborted, don't treat it as an error if (abortController.signal.aborted) { console.log("Operation was cancelled"); - return; + return newConnectorName; } const shouldRollbackConnectorFile = @@ -390,10 +428,7 @@ export async function submitAddConnectorForm( } finally { // Mark the submission as completed but keep the connector name around // so a subsequent "Save Anyway" can still reuse the same connector file - const submission = connectorSubmissions.get(uniqueConnectorSubmissionKey); - if (submission) { - submission.completed = true; - } + connectorSubmissions.delete(uniqueConnectorSubmissionKey); } })(); @@ -401,27 +436,35 @@ export async function submitAddConnectorForm( connectorSubmissions.set(uniqueConnectorSubmissionKey, { promise: submissionPromise, connectorName: newConnectorName, - completed: false, }); // Wait for the submission to complete - await submissionPromise; + const resolvedConnectorName = await submissionPromise; + return resolvedConnectorName; } export async function submitAddSourceForm( queryClient: QueryClient, connector: V1ConnectorDriver, formValues: AddDataFormValues, + connectorInstanceName?: string, ): Promise { const instanceId = get(runtime).instanceId; await beforeSubmitForm(instanceId, connector); - const newSourceName = formValues.name as string; const [rewrittenConnector, rewrittenFormValues] = prepareSourceFormData( connector, formValues, + { connectorInstanceName }, ); + const schema = getConnectorSchema(rewrittenConnector.name ?? ""); + const schemaSecretKeys = schema + ? getSchemaSecretKeys(schema, { step: "source" }) + : []; + const schemaStringKeys = schema + ? getSchemaStringKeys(schema, { step: "source" }) + : []; // Make a new .yaml file const newSourceFilePath = getFileAPIPathFromNameAndType( @@ -430,7 +473,10 @@ export async function submitAddSourceForm( ); await runtimeServicePutFile(instanceId, { path: newSourceFilePath, - blob: compileSourceYAML(rewrittenConnector, rewrittenFormValues), + blob: compileSourceYAML(rewrittenConnector, rewrittenFormValues, { + secretKeys: schemaSecretKeys, + stringKeys: schemaStringKeys, + }), create: true, createOnly: false, }); @@ -443,6 +489,8 @@ export async function submitAddSourceForm( rewrittenConnector, rewrittenFormValues, "source", + undefined, + { secretKeys: schemaSecretKeys }, ); // Make sure the file has reconciled before testing the connection diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 6bd2b9106f4..89b9b5051b7 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -6,8 +6,10 @@ import { findRadioEnumKey, getRadioEnumOptions, getRequiredFieldsByEnumValue, + getSchemaFieldMetaList, isStepMatch, } from "../../templates/schema-utils"; +import { getConnectorSchema } from "./connector-schemas"; /** * Returns true for undefined, null, empty string, or whitespace-only string. @@ -74,10 +76,16 @@ export function hasOnlyDsn( isConnectorForm: boolean, ): boolean { if (!isConnectorForm) return false; - const props = connector?.configProperties ?? []; - const hasDsn = props.some((p) => p.key === "dsn"); - const hasOthers = props.some((p) => p.key !== "dsn"); - return hasDsn && !hasOthers; + const schema = getConnectorSchema(connector?.name ?? ""); + if (schema) { + const fields = getSchemaFieldMetaList(schema, { step: "connector" }).filter( + (field) => !field.internal, + ); + const hasDsn = fields.some((field) => field.key === "dsn"); + const hasOthers = fields.some((field) => field.key !== "dsn"); + return hasDsn && !hasOthers; + } + return false; } /** @@ -94,15 +102,15 @@ export function isMultiStepConnectorDisabled( // For source step, gate on required fields from the JSON schema. const currentStep = step || (paramsFormValue?.__step as string | undefined); - if (currentStep === "source") { + if (currentStep === "source" || currentStep === "explorer") { const required = getRequiredFieldsForStep( schema, paramsFormValue, - "source", + currentStep, ); if (!required.length) return false; return !required.every((fieldId) => { - if (!isStepMatch(schema, fieldId, "source")) return true; + if (!isStepMatch(schema, fieldId, currentStep)) return true; const value = paramsFormValue[fieldId]; const errorsForField = paramsFormErrors[fieldId] as any; const hasErrors = Boolean(errorsForField?.length); @@ -113,6 +121,21 @@ export function isMultiStepConnectorDisabled( const authInfo = getRadioEnumOptions(schema); const options = authInfo?.options ?? []; const authKey = authInfo?.key || findRadioEnumKey(schema); + if (!authInfo || !options.length || !authKey) { + const required = getRequiredFieldsForStep( + schema, + paramsFormValue, + "connector", + ); + if (!required.length) return false; + return !required.every((fieldId) => { + if (!isStepMatch(schema, fieldId, "connector")) return true; + const value = paramsFormValue[fieldId]; + const errorsForField = paramsFormErrors[fieldId] as any; + const hasErrors = Boolean(errorsForField?.length); + return !isEmpty(value) && !hasErrors; + }); + } const methodFromForm = authKey && paramsFormValue?.[authKey] != null ? String(paramsFormValue[authKey]) diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts deleted file mode 100644 index 18fe8befa14..00000000000 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ /dev/null @@ -1,185 +0,0 @@ -import * as yup from "yup"; -import { - INVALID_NAME_MESSAGE, - VALID_NAME_PATTERN, -} from "../../entity-management/name-utils"; - -export const getYupSchema = { - https: yup.object().shape({ - path: yup - .string() - .matches(/^https?:\/\//, 'Path must start with "http(s)://"') - .required("Path is required"), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), - }), - - duckdb: yup.object().shape({ - path: yup.string().required("path is required"), - attach: yup.string().optional(), - }), - - motherduck: yup.object().shape({ - token: yup.string().required("Token is required"), - path: yup.string().required("Path is required"), - schema_name: yup.string().required("Schema name is required"), - }), - - sqlite: yup.object().shape({ - db: yup.string().required("db is required"), - table: yup.string().required("table is required"), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), - }), - - bigquery: yup.object().shape({ - project_id: yup.string(), - google_application_credentials: yup - .string() - .required("Google application credentials is required"), - }), - - postgres: yup.object().shape({ - dsn: yup.string().optional(), - host: yup.string().optional(), - port: yup.string().optional(), - user: yup.string().optional(), - password: yup.string().optional(), - dbname: yup.string().optional(), - sslmode: yup.string().optional(), - }), - - snowflake: yup.object().shape({ - dsn: yup.string().optional(), - account: yup.string().required("Account is required"), - user: yup.string().required("Username is required"), - password: yup.string().required("Password is required"), - database: yup.string().optional(), - schema: yup.string().optional(), - warehouse: yup.string().optional(), - role: yup.string().optional(), - }), - - salesforce: yup.object().shape({ - soql: yup.string().required("soql is required"), - sobject: yup.string().required("sobject is required"), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), - }), - - athena: yup.object().shape({ - aws_access_key_id: yup.string().required("AWS access key ID is required"), - aws_secret_access_key: yup - .string() - .required("AWS secret access key is required"), - output_location: yup.string().required("S3 URI is required"), - }), - - redshift: yup.object().shape({ - aws_access_key_id: yup.string().required("AWS access key ID is required"), - aws_secret_access_key: yup - .string() - .required("AWS secret access key is required"), - workgroup: yup.string().optional(), - region: yup.string().optional(), // TODO: add validation - database: yup.string().required("database name is required"), - }), - - mysql: yup.object().shape({ - dsn: yup.string().optional(), - user: yup.string().optional(), - password: yup.string().optional(), - host: yup.string().optional(), - port: yup.string().optional(), - database: yup.string().optional(), - sslmode: yup.string().optional(), - }), - - clickhouse: yup.object().shape({ - dsn: yup.string().optional(), - managed: yup.boolean(), - host: yup.string(), - // .required("Host is required") - // .matches( - // /^(?!https?:\/\/)[a-zA-Z0-9.-]+$/, - // "Do not prefix the host with `http(s)://`", // It will be added by the runtime - // ), - port: yup - .string() // Purposefully using a string input, not a numeric input - .matches(/^\d+$/, "Port must be a number"), - username: yup.string(), - password: yup.string(), - cluster: yup.string(), - ssl: yup.boolean(), - name: yup.string(), // Required for typing - // User-provided connector names requires a little refactor. Commenting out for now. - // name: yup - // .string() - // .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - // .required("Connector name is required"), - }), - - druid: yup.object().shape({ - host: yup - .string() - .required("Host is required") - .matches( - /^(?!https?:\/\/)[a-zA-Z0-9.-]+$/, - "Do not prefix the host with `http(s)://`", // It will be added by the runtime - ), - port: yup - .string() // Purposefully using a string input, not a numeric input - .matches(/^\d+$/, "Port must be a number"), - username: yup.string(), - password: yup.string(), - ssl: yup.boolean(), - name: yup.string(), // Required for typing - // User-provided connector names requires a little refactor. Commenting out for now. - // name: yup - // .string() - // .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - // .required("Connector name is required"), - }), - - pinot: yup.object().shape({ - broker_host: yup - .string() - .required("Broker host is required") - .matches( - /^(?!https?:\/\/)[a-zA-Z0-9.-]+$/, - "Do not prefix the host with `http(s)://`", // It will be added by the runtime - ), - broker_port: yup - .string() // Purposefully using a string input, not a numeric input - .matches(/^\d+$/, "Port must be a number"), - controller_host: yup - .string() - .required("Controller host is required") - .matches( - /^(?!https?:\/\/)[a-zA-Z0-9.-]+$/, - "Do not prefix the host with `http(s)://`", // It will be added by the runtime - ), - controller_port: yup - .string() // Purposefully using a string input, not a numeric input - .matches(/^\d+$/, "Port must be a number"), - username: yup.string(), - password: yup.string(), - ssl: yup.boolean(), - name: yup.string(), // Required for typing - // User-provided connector names requires a little refactor. Commenting out for now. - // name: yup - // .string() - // .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - // .required("Connector name is required"), - }), -}; - -export const dsnSchema = yup.object().shape({ - dsn: yup.string().required("DSN is required"), -}); diff --git a/web-common/src/features/sources/sourceUtils.ts b/web-common/src/features/sources/sourceUtils.ts index d181415b56e..5137b1d6548 100644 --- a/web-common/src/features/sources/sourceUtils.ts +++ b/web-common/src/features/sources/sourceUtils.ts @@ -1,12 +1,16 @@ import { extractFileExtension } from "@rilldata/web-common/features/entity-management/file-path-utils"; -import { - ConnectorDriverPropertyType, - type ConnectorDriverProperty, - type V1ConnectorDriver, - type V1Source, +import type { + V1ConnectorDriver, + V1Source, } from "@rilldata/web-common/runtime-client"; import { makeDotEnvConnectorKey } from "../connectors/code-utils"; import { sanitizeEntityName } from "../entity-management/name-utils"; +import { getConnectorSchema } from "./modal/connector-schemas"; +import { + getSchemaFieldMetaList, + getSchemaSecretKeys, + getSchemaStringKeys, +} from "../templates/schema-utils"; // Helper text that we put at the top of every Model YAML file const SOURCE_MODEL_FILE_TOP = `# Model YAML @@ -18,20 +22,26 @@ materialize: true`; export function compileSourceYAML( connector: V1ConnectorDriver, formValues: Record, + opts?: { secretKeys?: string[]; stringKeys?: string[] }, ) { + const schema = getConnectorSchema(connector.name ?? ""); + // Get the secret property keys const secretPropertyKeys = - connector.sourceProperties - ?.filter((property) => property.secret) - .map((property) => property.key) || []; + opts?.secretKeys ?? + (schema ? getSchemaSecretKeys(schema, { step: "source" }) : []); // Get the string property keys const stringPropertyKeys = - connector.sourceProperties - ?.filter( - (property) => property.type === ConnectorDriverPropertyType.TYPE_STRING, - ) - .map((property) => property.key) || []; + opts?.stringKeys ?? + (schema ? getSchemaStringKeys(schema, { step: "source" }) : []); + + const formatSqlBlock = (sql: string, indent: string) => + `sql: |\n${sql + .split("\n") + .map((line) => `${indent}${line}`) + .join("\n")}`; + const trimSqlForDev = (sql: string) => sql.trim().replace(/;+\s*$/, ""); // Compile key value pairs const compiledKeyValues = Object.keys(formValues) @@ -58,10 +68,7 @@ export function compileSourceYAML( if (key === "sql") { // For SQL, we want to use a multi-line string - return `${key}: |\n ${value - .split("\n") - .map((line) => `${line}`) - .join("\n")}`; + return formatSqlBlock(value, " "); } const isStringProperty = stringPropertyKeys.includes(key); @@ -73,9 +80,21 @@ export function compileSourceYAML( }) .join("\n"); + const devSection = + connector.implementsWarehouse && + connector.name !== "redshift" && + typeof formValues.sql === "string" && + formValues.sql.trim() + ? `\n\ndev:\n ${formatSqlBlock( + `${trimSqlForDev(formValues.sql)} limit 10000`, + " ", + )}` + : ""; + return ( `${SOURCE_MODEL_FILE_TOP}\n\nconnector: ${connector.name}\n\n` + - compiledKeyValues + compiledKeyValues + + devSection ); } @@ -156,9 +175,13 @@ export function getFileTypeFromPath(fileName) { export function maybeRewriteToDuckDb( connector: V1ConnectorDriver, formValues: Record, + options?: { connectorInstanceName?: string }, ): [V1ConnectorDriver, Record] { // Create a copy of the connector, so that we don't overwrite the original const connectorCopy = { ...connector }; + const connectorInstanceName = + options?.connectorInstanceName?.trim() || undefined; + const secretConnectorName = connectorInstanceName || connector.name || ""; switch (connector.name) { case "s3": @@ -166,8 +189,15 @@ export function maybeRewriteToDuckDb( case "https": case "azure": // Ensure DuckDB creates a temporary secret for the original connector - if (!formValues.create_secrets_from_connectors) { - formValues.create_secrets_from_connectors = connector.name; + if (secretConnectorName) { + if (connectorInstanceName) { + if (!formValues.create_secrets_from_connectors) { + formValues.create_secrets_from_connectors = secretConnectorName; + } + } else { + // When skipping connector creation, force the default driver name. + formValues.create_secrets_from_connectors = secretConnectorName; + } } // falls through to rewrite as DuckDB case "local_file": @@ -176,13 +206,6 @@ export function maybeRewriteToDuckDb( formValues.sql = buildDuckDbQuery(formValues.path as string); delete formValues.path; - connectorCopy.sourceProperties = [ - { - key: "sql", - type: ConnectorDriverPropertyType.TYPE_STRING, - }, - ]; - break; case "sqlite": connectorCopy.name = "duckdb"; @@ -193,13 +216,6 @@ export function maybeRewriteToDuckDb( delete formValues.db; delete formValues.table; - connectorCopy.sourceProperties = [ - { - key: "sql", - type: ConnectorDriverPropertyType.TYPE_STRING, - }, - ]; - break; } @@ -213,6 +229,7 @@ export function maybeRewriteToDuckDb( export function prepareSourceFormData( connector: V1ConnectorDriver, formValues: Record, + options?: { connectorInstanceName?: string }, ): [V1ConnectorDriver, Record] { // Create a copy of form values to avoid mutating the original const processedValues = { ...formValues }; @@ -222,9 +239,12 @@ export function prepareSourceFormData( // Strip connector configuration keys from the source form values to prevent // leaking connector-level fields (e.g., credentials) into the model file. - if (connector.configProperties) { + const schema = getConnectorSchema(connector.name ?? ""); + if (schema) { const connectorPropertyKeys = new Set( - connector.configProperties.map((p) => p.key).filter(Boolean), + getSchemaFieldMetaList(schema, { step: "connector" }) + .filter((field) => !field.internal) + .map((field) => field.key), ); for (const key of Object.keys(processedValues)) { if (connectorPropertyKeys.has(key)) { @@ -234,11 +254,12 @@ export function prepareSourceFormData( } // Handle placeholder values for required source properties - if (connector.sourceProperties) { - for (const prop of connector.sourceProperties) { - if (prop.key && prop.required && !(prop.key in processedValues)) { - if (prop.placeholder) { - processedValues[prop.key] = prop.placeholder; + if (schema) { + const sourceFields = getSchemaFieldMetaList(schema, { step: "source" }); + for (const field of sourceFields) { + if (field.required && !(field.key in processedValues)) { + if (field.placeholder) { + processedValues[field.key] = field.placeholder; } } } @@ -248,6 +269,7 @@ export function prepareSourceFormData( const [rewrittenConnector, rewrittenFormValues] = maybeRewriteToDuckDb( connector, processedValues, + options, ); return [rewrittenConnector, rewrittenFormValues]; @@ -276,34 +298,3 @@ export function formatConnectorType(source: V1Source) { return source?.state?.connector ?? ""; } } - -/** - * Extracts initial form values from connector property specs, using the Default field if present. - * @param properties Array of property specs (e.g., connector.configProperties) - * @returns Object mapping property keys to their default values - */ -export function getInitialFormValuesFromProperties( - properties: Array, -) { - const initialValues: Record = {}; - for (const prop of properties) { - // Only set if default is not undefined/null/empty string - if ( - prop.key && - prop.default !== undefined && - prop.default !== null && - prop.default !== "" - ) { - let value: any = prop.default; - if (prop.type === ConnectorDriverPropertyType.TYPE_NUMBER) { - // NOTE: store number type prop as String, not Number, so that we can use the same form for both number and string properties - // See `yupSchemas.ts` for more details - value = String(value); - } else if (prop.type === ConnectorDriverPropertyType.TYPE_BOOLEAN) { - value = value === "true" || value === true; - } - initialValues[prop.key] = value; - } - } - return initialValues; -} diff --git a/web-common/src/features/templates/JSONSchemaFormRenderer.svelte b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte index 6b6bcfa294b..a3bffcf2b12 100644 --- a/web-common/src/features/templates/JSONSchemaFormRenderer.svelte +++ b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte @@ -1,5 +1,7 @@ -
- {#if schema} - {#each renderOrder as [key, prop]} - {#if isRadioEnum(prop)} -
- {#if prop.title} -
{prop.title}
- {/if} - - - {#if groupedFields.get(key)} - {#each getGroupedFieldsForOption(key, option.value) as [childKey, childProp]} +{#if schema} + {#each renderOrder as [key, prop]} + {#if isRadioEnum(prop)} +
+ {#if prop.title} +
{prop.title}
+ {/if} + + + {#if groupedFields.get(key)} + {#each getGroupedFieldsForOption(key, option.value) as [childKey, childProp]} +
+ {#if isTabsEnum(childProp)} + {@const childOptions = tabOptions(childProp)} + {#if childProp.title} +
+ {childProp.title} +
+ {/if} + + {#each childOptions as childOption} + + {#if tabGroupedFields.get(childKey)} + {#each getTabFieldsForOption(childKey, childOption.value) as [tabKey, tabProp]} +
+ +
+ {/each} + {/if} + + {/each} + + {:else} + + {/if} +
+ {/each} + {/if} + + +
+ {:else if isTabsEnum(prop)} + {@const options = tabOptions(prop)} +
+ {#if prop.title} +
{prop.title}
+ {/if} + + {#each options as option} + + {#if tabGroupedFields.get(key)} + {#each getTabFieldsForOption(key, option.value) as [childKey, childProp]}
{/each} {/if} - - -
- {:else} -
- -
- {/if} - {/each} - {/if} - + + {/each} + +
+ {:else} +
+ +
+ {/if} + {/each} +{/if} diff --git a/web-common/src/features/templates/SchemaField.svelte b/web-common/src/features/templates/SchemaField.svelte index 490287bb05a..2d76ac6e8b8 100644 --- a/web-common/src/features/templates/SchemaField.svelte +++ b/web-common/src/features/templates/SchemaField.svelte @@ -1,5 +1,6 @@ -{#if prop["x-display"] === "file" || prop.format === "file"} +{#if prop["x-informational"]} + +{:else if prop["x-display"] === "file" || prop.format === "file"} onStringInputChange(e)} alwaysShowError /> diff --git a/web-common/src/features/templates/schema-utils.ts b/web-common/src/features/templates/schema-utils.ts index a4544947297..09617fd588a 100644 --- a/web-common/src/features/templates/schema-utils.ts +++ b/web-common/src/features/templates/schema-utils.ts @@ -1,5 +1,6 @@ import type { JSONSchemaConditional, + JSONSchemaField, MultiStepFormSchema, } from "./schemas/types"; @@ -20,7 +21,7 @@ export function isStepMatch( if (!prop) return false; if (!step) return true; const propStep = prop["x-step"]; - if (!propStep) return true; + if (!propStep) return step !== "explorer"; return propStep === step; } @@ -50,6 +51,149 @@ export function getFieldLabel( return schema.properties?.[key]?.title || key; } +export type SchemaFieldMeta = { + key: string; + type?: "string" | "number" | "boolean" | "object"; + displayName: string; + description?: string; + placeholder?: string; + hint?: string; + secret?: boolean; + docsUrl?: string; + required?: boolean; + default?: string | number | boolean; + enum?: Array; + informational?: boolean; + internal?: boolean; +}; + +export function getSchemaFieldMetaList( + schema: MultiStepFormSchema, + opts?: { step?: "connector" | "source" | string }, +): SchemaFieldMeta[] { + const properties = schema.properties ?? {}; + const required = new Set( + (schema.required ?? []).filter((key) => + isStepMatch(schema, key, opts?.step), + ), + ); + + return Object.entries(properties) + .filter(([key]) => isStepMatch(schema, key, opts?.step)) + .map(([key, prop]) => ({ + key, + type: prop.type, + displayName: prop.title ?? key, + description: prop.description, + placeholder: prop["x-placeholder"], + hint: prop["x-hint"], + secret: Boolean(prop["x-secret"]), + docsUrl: prop["x-docs-url"], + required: required.has(key), + default: prop.default, + enum: prop.enum, + informational: Boolean(prop["x-informational"]), + internal: Boolean(prop["x-internal"]), + })); +} + +export function getSchemaInitialValues( + schema: MultiStepFormSchema, + opts?: { step?: "connector" | "source" | string }, +): Record { + const initial: Record = {}; + const properties = schema.properties ?? {}; + + for (const [key, prop] of Object.entries(properties)) { + if (!isStepMatch(schema, key, opts?.step)) continue; + if (prop.default !== undefined && prop.default !== null) { + initial[key] = prop.default; + continue; + } + if ( + prop.enum?.length && + (prop["x-display"] === "radio" || prop["x-display"] === "tabs") + ) { + initial[key] = String(prop.enum[0]); + } + } + + return initial; +} + +export function getRequiredFieldsForValues( + schema: MultiStepFormSchema, + values: Record, + step?: "connector" | "source" | string, +): Set { + const required = new Set(); + (schema.required ?? []).forEach((key) => { + if (isStepMatch(schema, key, step)) required.add(key); + }); + + for (const conditional of schema.allOf ?? []) { + const condition = conditional.if?.properties; + const matches = matchesCondition(condition, values); + const branch = matches ? conditional.then : conditional.else; + branch?.required?.forEach((key) => { + if (isStepMatch(schema, key, step)) required.add(key); + }); + } + + return required; +} + +export function getSchemaSecretKeys( + schema: MultiStepFormSchema, + opts?: { step?: "connector" | "source" | string }, +): string[] { + const properties = schema.properties ?? {}; + return Object.entries(properties) + .filter( + ([key, prop]) => + isStepMatch(schema, key, opts?.step) && Boolean(prop["x-secret"]), + ) + .map(([key]) => key); +} + +export function getSchemaStringKeys( + schema: MultiStepFormSchema, + opts?: { step?: "connector" | "source" | string }, +): string[] { + const properties = schema.properties ?? {}; + return Object.entries(properties) + .filter( + ([key, prop]) => + isStepMatch(schema, key, opts?.step) && prop.type === "string", + ) + .map(([key]) => key); +} + +export function filterSchemaInternalValues( + schema: MultiStepFormSchema, + values: Record, + opts?: { step?: "connector" | "source" | string }, +): Record { + const properties = schema.properties ?? {}; + return Object.fromEntries( + Object.entries(values).filter(([key]) => { + const prop = properties[key] as JSONSchemaField | undefined; + if (!prop) return false; + if (!isStepMatch(schema, key, opts?.step)) return false; + return !prop["x-internal"]; + }), + ); +} + +export function filterSchemaValuesForSubmit( + schema: MultiStepFormSchema, + values: Record, + opts?: { step?: "connector" | "source" | string }, +): Record { + const tabFiltered = filterValuesByTabGroups(schema, values, opts); + return filterSchemaInternalValues(schema, tabFiltered, opts); +} + export function findRadioEnumKey(schema: MultiStepFormSchema): string | null { if (!schema.properties) return null; for (const [key, value] of Object.entries(schema.properties)) { @@ -142,6 +286,17 @@ export function getRequiredFieldsByEnumValue( return result; } +function matchesCondition( + condition: Record | undefined, + values: Record, +) { + if (!condition || !Object.keys(condition).length) return false; + return Object.entries(condition).every(([depKey, def]) => { + if (def.const === undefined || def.const === null) return false; + return String(values?.[depKey]) === String(def.const); + }); +} + function matchesEnumCondition( conditional: JSONSchemaConditional, enumKey: string, @@ -152,3 +307,28 @@ function matchesEnumCondition( if (constValue === undefined || constValue === null) return false; return String(constValue) === value; } + +function filterValuesByTabGroups( + schema: MultiStepFormSchema, + values: Record, + opts?: { step?: "connector" | "source" | string }, +) { + const properties = schema.properties ?? {}; + const result = { ...values }; + + for (const [key, prop] of Object.entries(properties)) { + if (!isStepMatch(schema, key, opts?.step)) continue; + if (prop["x-display"] !== "tabs") continue; + const tabGroups = prop["x-tab-group"]; + if (!tabGroups) continue; + const selected = String(values?.[key] ?? ""); + const active = tabGroups[selected] ?? []; + const allChildKeys = new Set(Object.values(tabGroups).flat()); + for (const childKey of allChildKeys) { + if (active.includes(childKey)) continue; + delete result[childKey]; + } + } + + return result; +} diff --git a/web-common/src/features/templates/schemas/athena.ts b/web-common/src/features/templates/schemas/athena.ts new file mode 100644 index 00000000000..c4ea75b6fd7 --- /dev/null +++ b/web-common/src/features/templates/schemas/athena.ts @@ -0,0 +1,56 @@ +import type { MultiStepFormSchema } from "./types"; + +export const athenaSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + aws_access_key_id: { + type: "string", + title: "AWS access key ID", + description: "AWS access key ID used to authenticate to Athena", + "x-placeholder": "your_access_key_id", + "x-secret": true, + }, + aws_secret_access_key: { + type: "string", + title: "AWS secret access key", + description: "AWS secret access key paired with the access key ID", + "x-placeholder": "your_secret_access_key", + "x-secret": true, + }, + output_location: { + type: "string", + title: "S3 output location", + description: + "S3 URI where Athena should write query results (e.g., s3://bucket/path/)", + pattern: "^s3://.+", + errorMessage: { + pattern: "Must be an S3 URI (e.g., s3://bucket/path/)", + }, + "x-placeholder": "s3://bucket-name/path/", + }, + sql: { + type: "string", + title: "SQL", + description: "SQL query to run against your warehouse", + "x-display": "textarea", + "x-placeholder": "Input SQL", + "x-step": "explorer", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z0-9_]+$", + "x-placeholder": "my_model", + "x-step": "explorer", + }, + }, + required: [ + "aws_access_key_id", + "aws_secret_access_key", + "output_location", + "sql", + "name", + ], +}; diff --git a/web-common/src/features/templates/schemas/azure.ts b/web-common/src/features/templates/schemas/azure.ts index db741448076..bf102af1764 100644 --- a/web-common/src/features/templates/schemas/azure.ts +++ b/web-common/src/features/templates/schemas/azure.ts @@ -23,6 +23,7 @@ export const azureSchema: MultiStepFormSchema = { "Provide the storage account name and SAS token.", "Access publicly readable blobs without credentials.", ], + "x-internal": true, "x-grouped-fields": { connection_string: ["azure_storage_connection_string"], account_key: ["azure_storage_account", "azure_storage_key"], diff --git a/web-common/src/features/templates/schemas/bigquery.ts b/web-common/src/features/templates/schemas/bigquery.ts new file mode 100644 index 00000000000..8e524e07282 --- /dev/null +++ b/web-common/src/features/templates/schemas/bigquery.ts @@ -0,0 +1,42 @@ +import type { MultiStepFormSchema } from "./types"; + +export const bigquerySchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + google_application_credentials: { + type: "string", + title: "GCP credentials", + description: "Service account JSON (uploaded or pasted)", + format: "file", + "x-display": "file", + "x-accept": ".json", + "x-secret": true, + }, + project_id: { + type: "string", + title: "Project ID", + description: "Google Cloud project ID to use for queries", + "x-placeholder": "my-project", + "x-hint": + "If empty, Rill will use the project ID from your credentials when available.", + }, + sql: { + type: "string", + title: "SQL", + description: "SQL query to run against your warehouse", + "x-display": "textarea", + "x-placeholder": "Input SQL", + "x-step": "explorer", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z0-9_]+$", + "x-placeholder": "my_model", + "x-step": "explorer", + }, + }, + required: ["google_application_credentials", "project_id", "sql", "name"], +}; diff --git a/web-common/src/features/templates/schemas/clickhouse.ts b/web-common/src/features/templates/schemas/clickhouse.ts new file mode 100644 index 00000000000..7376ee9b7f3 --- /dev/null +++ b/web-common/src/features/templates/schemas/clickhouse.ts @@ -0,0 +1,226 @@ +import type { MultiStepFormSchema } from "./types"; + +export const clickhouseSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + connector_type: { + type: "string", + title: "Connection type", + enum: ["rill-managed", "self-hosted", "clickhouse-cloud"], + default: "self-hosted", + "x-display": "radio", + "x-enum-labels": [ + "Rill-managed ClickHouse", + "Self-hosted ClickHouse", + "ClickHouse Cloud", + ], + "x-internal": true, + "x-grouped-fields": { + "rill-managed": ["managed"], + "self-hosted": ["connection_mode"], + "clickhouse-cloud": ["connection_mode"], + }, + "x-step": "connector", + }, + connection_mode: { + type: "string", + enum: ["parameters", "dsn"], + default: "parameters", + "x-display": "tabs", + "x-enum-labels": ["Enter parameters", "Enter connection string"], + "x-internal": true, + "x-tab-group": { + parameters: [ + "host", + "port", + "username", + "password", + "database", + "cluster", + "ssl", + ], + dsn: ["dsn"], + }, + "x-visible-if": { + connector_type: ["self-hosted", "clickhouse-cloud"], + }, + "x-step": "connector", + }, + dsn: { + type: "string", + title: "Connection string", + description: + "DSN connection string (use instead of individual host/port/user settings)", + "x-placeholder": + "clickhouse://localhost:9000?username=default&password=password", + "x-visible-if": { + connector_type: ["self-hosted", "clickhouse-cloud"], + }, + "x-step": "connector", + }, + managed: { + type: "boolean", + title: "Managed", + description: + "This option uses ClickHouse as an OLAP engine with Rill-managed infrastructure. No additional configuration is required - Rill will handle the setup and management of your ClickHouse instance.", + default: false, + "x-informational": true, + "x-visible-if": { + connector_type: "rill-managed", + }, + "x-step": "connector", + }, + host: { + type: "string", + title: "Host", + description: "Hostname or IP address of the ClickHouse server", + "x-placeholder": + "your-instance.clickhouse.cloud or your.clickhouse.server.com", + "x-hint": + "Your ClickHouse hostname (e.g., your-instance.clickhouse.cloud or your-server.com)", + "x-visible-if": { + connector_type: ["self-hosted", "clickhouse-cloud"], + }, + "x-step": "connector", + }, + port: { + type: "string", + title: "Port", + description: "Port number of the ClickHouse server", + pattern: "^\\d+$", + errorMessage: { pattern: "Port must be a number" }, + default: "9000", + "x-placeholder": "9000", + "x-visible-if": { + connector_type: ["self-hosted", "clickhouse-cloud"], + }, + "x-step": "connector", + }, + username: { + type: "string", + title: "Username", + description: "Username to connect to the ClickHouse server", + default: "default", + "x-placeholder": "default", + "x-visible-if": { + connector_type: ["self-hosted", "clickhouse-cloud"], + }, + "x-step": "connector", + }, + password: { + type: "string", + title: "Password", + description: "Password to connect to the ClickHouse server", + "x-placeholder": "Database password", + "x-secret": true, + "x-visible-if": { + connector_type: ["self-hosted", "clickhouse-cloud"], + }, + "x-step": "connector", + }, + database: { + type: "string", + title: "Database", + description: "Name of the ClickHouse database to connect to", + default: "default", + "x-placeholder": "default", + "x-visible-if": { + connector_type: ["self-hosted", "clickhouse-cloud"], + }, + "x-step": "connector", + }, + cluster: { + type: "string", + title: "Cluster", + description: + "Cluster name. If set, models are created as distributed tables.", + "x-placeholder": "Cluster name", + "x-visible-if": { + connector_type: ["self-hosted", "clickhouse-cloud"], + }, + "x-step": "connector", + }, + ssl: { + type: "boolean", + title: "SSL", + description: "Use SSL to connect to the ClickHouse server", + default: true, + "x-visible-if": { + connector_type: ["self-hosted", "clickhouse-cloud"], + }, + "x-step": "connector", + }, + }, + required: ["connector_type"], + allOf: [ + { + if: { properties: { connector_type: { const: "rill-managed" } } }, + then: { + required: ["managed"], + properties: { + managed: { const: true }, + }, + }, + }, + { + if: { + properties: { + connector_type: { const: "self-hosted" }, + connection_mode: { const: "parameters" }, + }, + }, + then: { + required: ["host", "username"], + properties: { + managed: { const: false }, + ssl: { default: true }, + }, + }, + }, + { + if: { + properties: { + connector_type: { const: "self-hosted" }, + connection_mode: { const: "dsn" }, + }, + }, + then: { + required: ["dsn"], + properties: { + managed: { const: false }, + }, + }, + }, + { + if: { + properties: { + connector_type: { const: "clickhouse-cloud" }, + connection_mode: { const: "parameters" }, + }, + }, + then: { + required: ["host", "username", "ssl"], + properties: { + managed: { const: false }, + port: { default: "8443" }, + ssl: { const: true }, + }, + }, + }, + { + if: { + properties: { + connector_type: { const: "clickhouse-cloud" }, + connection_mode: { const: "dsn" }, + }, + }, + then: { + required: ["dsn"], + properties: { + managed: { const: false }, + }, + }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/druid.ts b/web-common/src/features/templates/schemas/druid.ts new file mode 100644 index 00000000000..d7d69c79d8c --- /dev/null +++ b/web-common/src/features/templates/schemas/druid.ts @@ -0,0 +1,74 @@ +import type { MultiStepFormSchema } from "./types"; + +export const druidSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + connection_mode: { + type: "string", + title: "Connection method", + enum: ["parameters", "dsn"], + default: "parameters", + "x-display": "tabs", + "x-enum-labels": ["Enter parameters", "Enter connection string"], + "x-internal": true, + "x-tab-group": { + parameters: ["host", "port", "username", "password", "ssl"], + dsn: ["dsn"], + }, + }, + dsn: { + type: "string", + title: "Connection string", + description: + "Full Druid SQL/Avatica endpoint, e.g. https://host:8888/druid/v2/sql/avatica-protobuf?authentication=BASIC&avaticaUser=user&avaticaPassword=pass", + "x-placeholder": + "https://example.com/druid/v2/sql/avatica-protobuf?authentication=BASIC&avaticaUser=user&avaticaPassword=pass", + "x-secret": true, + }, + host: { + type: "string", + title: "Host", + description: "Druid host or IP", + "x-placeholder": "localhost", + }, + port: { + type: "string", + title: "Port", + description: "Druid port", + pattern: "^\\d+$", + errorMessage: { pattern: "Port must be a number" }, + "x-placeholder": "8888", + }, + username: { + type: "string", + title: "Username", + description: "Druid username", + "x-placeholder": "default", + }, + password: { + type: "string", + title: "Password", + description: "Druid password", + "x-placeholder": "password", + "x-secret": true, + }, + ssl: { + type: "boolean", + title: "SSL", + description: "Use SSL for the connection", + default: true, + }, + }, + required: [], + oneOf: [ + { + title: "Use connection string", + required: ["dsn"], + }, + { + title: "Use individual parameters", + required: ["host", "ssl"], + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/duckdb.ts b/web-common/src/features/templates/schemas/duckdb.ts new file mode 100644 index 00000000000..3e893aa79f3 --- /dev/null +++ b/web-common/src/features/templates/schemas/duckdb.ts @@ -0,0 +1,15 @@ +import type { MultiStepFormSchema } from "./types"; + +export const duckdbSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + path: { + type: "string", + title: "Path", + description: "Path to external DuckDB database", + "x-placeholder": "/path/to/main.db", + }, + }, + required: ["path"], +}; diff --git a/web-common/src/features/templates/schemas/gcs.ts b/web-common/src/features/templates/schemas/gcs.ts index a4e1125f1ff..e28e98c4843 100644 --- a/web-common/src/features/templates/schemas/gcs.ts +++ b/web-common/src/features/templates/schemas/gcs.ts @@ -17,6 +17,7 @@ export const gcsSchema: MultiStepFormSchema = { "Use HMAC access key and secret for S3-compatible authentication.", "Access publicly readable buckets without credentials.", ], + "x-internal": true, "x-grouped-fields": { credentials: ["google_application_credentials"], hmac: ["key_id", "secret"], @@ -32,6 +33,7 @@ export const gcsSchema: MultiStepFormSchema = { format: "file", "x-display": "file", "x-accept": ".json", + "x-secret": true, "x-step": "connector", "x-visible-if": { auth_method: "credentials" }, }, diff --git a/web-common/src/features/templates/schemas/https.ts b/web-common/src/features/templates/schemas/https.ts new file mode 100644 index 00000000000..157b76b3eff --- /dev/null +++ b/web-common/src/features/templates/schemas/https.ts @@ -0,0 +1,27 @@ +import type { MultiStepFormSchema } from "./types"; + +export const httpsSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + path: { + type: "string", + title: "Path", + description: "HTTP/HTTPS URL to the remote file", + pattern: "^https?://.+", + errorMessage: { + pattern: "Path must start with http:// or https://", + }, + "x-placeholder": "https://example.com/file.csv", + "x-step": "source", + }, + name: { + type: "string", + title: "Source name", + description: "Name of the source", + "x-placeholder": "my_new_source", + "x-step": "source", + }, + }, + required: ["path", "name"], +}; diff --git a/web-common/src/features/templates/schemas/local_file.ts b/web-common/src/features/templates/schemas/local_file.ts new file mode 100644 index 00000000000..2fec6b68784 --- /dev/null +++ b/web-common/src/features/templates/schemas/local_file.ts @@ -0,0 +1,23 @@ +import type { MultiStepFormSchema } from "./types"; + +export const localFileSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + path: { + type: "string", + title: "Path", + description: "Local file path or glob (relative to project root)", + "x-placeholder": "data/*.parquet", + "x-step": "source", + }, + name: { + type: "string", + title: "Source name", + description: "Name for the source", + "x-placeholder": "my_new_source", + "x-step": "source", + }, + }, + required: ["path", "name"], +}; diff --git a/web-common/src/features/templates/schemas/motherduck.ts b/web-common/src/features/templates/schemas/motherduck.ts new file mode 100644 index 00000000000..7597ae50e4e --- /dev/null +++ b/web-common/src/features/templates/schemas/motherduck.ts @@ -0,0 +1,28 @@ +import type { MultiStepFormSchema } from "./types"; + +export const motherduckSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + path: { + type: "string", + title: "Path", + description: "MotherDuck database path (prefix with md:)", + "x-placeholder": "md:my_db", + }, + token: { + type: "string", + title: "Token", + description: "MotherDuck token", + "x-placeholder": "your_motherduck_token", + "x-secret": true, + }, + schema_name: { + type: "string", + title: "Schema name", + description: "Default schema to use", + "x-placeholder": "main", + }, + }, + required: ["path", "token", "schema_name"], +}; diff --git a/web-common/src/features/templates/schemas/mysql.ts b/web-common/src/features/templates/schemas/mysql.ts new file mode 100644 index 00000000000..da980e8aaf4 --- /dev/null +++ b/web-common/src/features/templates/schemas/mysql.ts @@ -0,0 +1,107 @@ +import type { MultiStepFormSchema } from "./types"; + +export const mysqlSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + connection_mode: { + type: "string", + title: "Connection method", + enum: ["parameters", "dsn"], + default: "parameters", + "x-display": "tabs", + "x-enum-labels": ["Enter parameters", "Enter connection string"], + "x-internal": true, + "x-tab-group": { + parameters: [ + "host", + "port", + "database", + "user", + "password", + "ssl-mode", + ], + dsn: ["dsn"], + }, + }, + dsn: { + type: "string", + title: "MySQL connection string", + description: + "Full DSN, e.g. mysql://user:password@host:3306/database?ssl-mode=REQUIRED", + "x-placeholder": "mysql://user:password@host:3306/database", + "x-secret": true, + "x-hint": + "Use DSN or fill host/user/password/database below (not both at once).", + }, + host: { + type: "string", + title: "Host", + description: "MySQL server hostname or IP", + "x-placeholder": "localhost", + }, + port: { + type: "string", + title: "Port", + description: "MySQL server port", + pattern: "^\\d+$", + errorMessage: { pattern: "Port must be a number" }, + default: "3306", + "x-placeholder": "3306", + }, + database: { + type: "string", + title: "Database", + description: "Database name", + "x-placeholder": "my_database", + }, + user: { + type: "string", + title: "Username", + description: "MySQL user", + "x-placeholder": "mysql", + }, + password: { + type: "string", + title: "Password", + description: "MySQL password", + "x-placeholder": "your_password", + "x-secret": true, + }, + "ssl-mode": { + type: "string", + title: "SSL mode", + description: "Use DISABLED, PREFERRED, or REQUIRED", + enum: ["DISABLED", "PREFERRED", "REQUIRED"], + "x-placeholder": "PREFERRED", + }, + }, + required: [], + oneOf: [ + { + title: "Use DSN", + required: ["dsn"], + not: { + anyOf: [ + { required: ["host"] }, + { required: ["database"] }, + { required: ["user"] }, + { required: ["password"] }, + { required: ["port"] }, + { required: ["ssl-mode"] }, + ], + }, + }, + { + title: "Use individual parameters", + required: ["host", "database", "user"], + }, + ], + allOf: [ + { + if: { properties: { connection_mode: { const: "dsn" } } }, + then: { required: ["dsn"] }, + else: { required: ["host", "database", "user"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/pinot.ts b/web-common/src/features/templates/schemas/pinot.ts new file mode 100644 index 00000000000..87dbb0eb1c4 --- /dev/null +++ b/web-common/src/features/templates/schemas/pinot.ts @@ -0,0 +1,96 @@ +import type { MultiStepFormSchema } from "./types"; + +export const pinotSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + connection_mode: { + type: "string", + title: "Connection method", + enum: ["parameters", "dsn"], + default: "parameters", + "x-display": "tabs", + "x-enum-labels": ["Enter parameters", "Enter connection string"], + "x-internal": true, + "x-tab-group": { + parameters: [ + "broker_host", + "broker_port", + "controller_host", + "controller_port", + "username", + "password", + "ssl", + ], + dsn: ["dsn"], + }, + }, + dsn: { + type: "string", + title: "Connection string", + description: + "Full Pinot connection string, e.g. http(s)://user:password@broker:8000?controller=host:9000", + "x-placeholder": + "https://username:password@localhost:8000?controller=localhost:9000", + "x-secret": true, + }, + broker_host: { + type: "string", + title: "Broker host", + description: "Pinot broker host", + "x-placeholder": "localhost", + }, + broker_port: { + type: "string", + title: "Broker port", + description: "Pinot broker port", + pattern: "^\\d+$", + errorMessage: { pattern: "Port must be a number" }, + "x-placeholder": "8000", + }, + controller_host: { + type: "string", + title: "Controller host", + description: "Pinot controller host", + "x-placeholder": "localhost", + }, + controller_port: { + type: "string", + title: "Controller port", + description: "Pinot controller port", + pattern: "^\\d+$", + errorMessage: { pattern: "Port must be a number" }, + "x-placeholder": "9000", + }, + username: { + type: "string", + title: "Username", + description: "Pinot username", + "x-placeholder": "default", + }, + password: { + type: "string", + title: "Password", + description: "Pinot password", + "x-placeholder": "password", + "x-secret": true, + }, + ssl: { + type: "boolean", + title: "SSL", + description: "Use SSL", + default: true, + }, + }, + required: [], + oneOf: [ + { + title: "Use connection string", + required: ["dsn"], + }, + { + title: "Use individual parameters", + required: ["broker_host", "controller_host", "ssl"], + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/postgres.ts b/web-common/src/features/templates/schemas/postgres.ts new file mode 100644 index 00000000000..aac96c2e258 --- /dev/null +++ b/web-common/src/features/templates/schemas/postgres.ts @@ -0,0 +1,116 @@ +import type { MultiStepFormSchema } from "./types"; + +export const postgresSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + connection_mode: { + type: "string", + title: "Connection method", + enum: ["parameters", "dsn"], + default: "parameters", + "x-display": "tabs", + "x-enum-labels": ["Enter parameters", "Enter connection string"], + "x-internal": true, + "x-tab-group": { + parameters: ["host", "port", "user", "password", "dbname", "sslmode"], + dsn: ["dsn"], + }, + }, + dsn: { + type: "string", + title: "Postgres connection string", + description: + "e.g. postgresql://user:password@host:5432/dbname?sslmode=require", + "x-placeholder": "postgresql://postgres:postgres@localhost:5432/postgres", + "x-secret": true, + "x-hint": + "Use a DSN or provide host/user/password/dbname below (but not both).", + }, + host: { + type: "string", + title: "Host", + description: "Postgres server hostname or IP", + "x-placeholder": "localhost", + }, + port: { + type: "string", + title: "Port", + description: "Postgres server port", + pattern: "^\\d+$", + errorMessage: { pattern: "Port must be a number" }, + default: "5432", + "x-placeholder": "5432", + }, + user: { + type: "string", + title: "Username", + description: "Postgres user", + "x-placeholder": "postgres", + }, + password: { + type: "string", + title: "Password", + description: "Postgres password", + "x-placeholder": "your_password", + "x-secret": true, + }, + dbname: { + type: "string", + title: "Database", + description: "Database name", + "x-placeholder": "postgres", + }, + sslmode: { + type: "string", + title: "SSL mode", + description: "Use disable, allow, prefer, require", + enum: ["disable", "allow", "prefer", "require"], + "x-placeholder": "require", + }, + }, + required: [], + oneOf: [ + { + title: "Use DSN", + required: ["dsn"], + not: { + anyOf: [ + { required: ["database_url"] }, + { required: ["host"] }, + { required: ["port"] }, + { required: ["user"] }, + { required: ["password"] }, + { required: ["dbname"] }, + { required: ["sslmode"] }, + ], + }, + }, + { + title: "Use Database URL", + required: ["database_url"], + not: { + anyOf: [ + { required: ["dsn"] }, + { required: ["host"] }, + { required: ["port"] }, + { required: ["user"] }, + { required: ["password"] }, + { required: ["dbname"] }, + { required: ["sslmode"] }, + ], + }, + }, + { + title: "Use individual parameters", + required: ["host", "user", "dbname"], + }, + ], + allOf: [ + { + if: { properties: { connection_mode: { const: "dsn" } } }, + then: { required: ["dsn"] }, + else: { required: ["host", "user", "dbname"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/redshift.ts b/web-common/src/features/templates/schemas/redshift.ts new file mode 100644 index 00000000000..1340f8a77bf --- /dev/null +++ b/web-common/src/features/templates/schemas/redshift.ts @@ -0,0 +1,70 @@ +import type { MultiStepFormSchema } from "./types"; + +export const redshiftSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + aws_access_key_id: { + type: "string", + title: "AWS access key ID", + description: "AWS access key ID", + "x-placeholder": "your_access_key_id", + "x-secret": true, + }, + aws_secret_access_key: { + type: "string", + title: "AWS secret access key", + description: "AWS secret access key", + "x-placeholder": "your_secret_access_key", + "x-secret": true, + }, + region: { + type: "string", + title: "AWS region", + description: "AWS region (e.g. us-east-1)", + "x-placeholder": "us-east-1", + }, + database: { + type: "string", + title: "Database", + description: "Redshift database name", + "x-placeholder": "dev", + }, + workgroup: { + type: "string", + title: "Workgroup", + description: "Redshift Serverless workgroup name", + "x-placeholder": "default", + }, + cluster_identifier: { + type: "string", + title: "Cluster identifier", + description: + "Redshift cluster identifier (use when not using serverless)", + "x-placeholder": "redshift-cluster-1", + }, + sql: { + type: "string", + title: "SQL", + description: "SQL query to run against your warehouse", + "x-display": "textarea", + "x-placeholder": "Input SQL", + "x-step": "explorer", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z0-9_]+$", + "x-placeholder": "my_model", + "x-step": "explorer", + }, + }, + required: [ + "aws_access_key_id", + "aws_secret_access_key", + "database", + "sql", + "name", + ], +}; diff --git a/web-common/src/features/templates/schemas/s3.ts b/web-common/src/features/templates/schemas/s3.ts index ea19e1f6f5a..7b394c72fa3 100644 --- a/web-common/src/features/templates/schemas/s3.ts +++ b/web-common/src/features/templates/schemas/s3.ts @@ -16,6 +16,7 @@ export const s3Schema: MultiStepFormSchema = { "Use AWS access key ID and secret access key.", "Access publicly readable buckets without credentials.", ], + "x-internal": true, "x-grouped-fields": { access_keys: [ "aws_access_key_id", diff --git a/web-common/src/features/templates/schemas/salesforce.ts b/web-common/src/features/templates/schemas/salesforce.ts new file mode 100644 index 00000000000..c60092f32fe --- /dev/null +++ b/web-common/src/features/templates/schemas/salesforce.ts @@ -0,0 +1,94 @@ +import type { MultiStepFormSchema } from "./types"; + +export const salesforceSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + soql: { + type: "string", + title: "SOQL", + description: "SOQL query to extract data", + "x-display": "textarea", + "x-placeholder": "SELECT Id, Name FROM Opportunity", + "x-step": "source", + }, + sobject: { + type: "string", + title: "SObject", + description: "Salesforce object to query", + "x-placeholder": "Opportunity", + "x-step": "source", + }, + queryAll: { + type: "boolean", + title: "Query all", + description: "Include deleted and archived records", + default: false, + "x-step": "source", + }, + username: { + type: "string", + title: "Username", + description: "Salesforce username (usually an email)", + "x-placeholder": "user@example.com", + }, + password: { + type: "string", + title: "Password", + description: + "Salesforce password, optionally followed by security token if required", + "x-placeholder": "your_password_or_password+token", + "x-secret": true, + }, + key: { + type: "string", + title: "JWT private key", + description: "PEM-formatted private key for JWT auth", + "x-display": "textarea", + "x-secret": true, + }, + client_id: { + type: "string", + title: "Connected App Client ID", + description: "Client ID (consumer key) for JWT auth", + "x-placeholder": "Connected App client ID", + }, + endpoint: { + type: "string", + title: "Login endpoint", + description: + "Salesforce login URL (e.g., login.salesforce.com or test.salesforce.com)", + "x-placeholder": "login.salesforce.com", + }, + name: { + type: "string", + title: "Source name", + description: "Name for the source", + "x-placeholder": "my_new_source", + "x-step": "source", + }, + }, + required: ["soql", "sobject", "name"], + allOf: [ + { + if: { + properties: { + key: { const: "" }, + }, + }, + then: { + required: ["username", "password", "endpoint"], + }, + }, + { + if: { + properties: { + key: { const: undefined }, + }, + }, + then: { + required: ["username", "password", "endpoint"], + }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/snowflake.ts b/web-common/src/features/templates/schemas/snowflake.ts new file mode 100644 index 00000000000..6de15b8adc3 --- /dev/null +++ b/web-common/src/features/templates/schemas/snowflake.ts @@ -0,0 +1,125 @@ +import type { MultiStepFormSchema } from "./types"; + +export const snowflakeSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + connection_mode: { + type: "string", + title: "Connection method", + enum: ["parameters", "dsn"], + default: "parameters", + "x-display": "tabs", + "x-enum-labels": ["Enter parameters", "Enter connection string"], + "x-internal": true, + "x-tab-group": { + parameters: [ + "account", + "user", + "password", + "privateKey", + "authenticator", + "database", + "schema", + "warehouse", + "role", + ], + dsn: ["dsn"], + }, + }, + dsn: { + type: "string", + title: "Snowflake connection string", + description: + "Full Snowflake DSN, e.g. @//?warehouse=&role=", + "x-placeholder": + "@//?warehouse=&role=", + "x-secret": true, + "x-hint": + "Use a full DSN or fill the fields below (not both). Include authenticator and privateKey for JWT if needed.", + }, + account: { + type: "string", + title: "Account identifier", + description: + "Snowflake account identifier (from your Snowflake URL, before .snowflakecomputing.com)", + "x-placeholder": "abc12345", + }, + user: { + type: "string", + title: "Username", + description: "Snowflake username", + "x-placeholder": "your_username", + }, + password: { + type: "string", + title: "Password", + description: + "Snowflake password (use JWT private key if password auth is disabled)", + "x-placeholder": "your_password", + "x-secret": true, + }, + privateKey: { + type: "string", + title: "Private key (JWT)", + description: + "URL-safe base64 or PEM private key for SNOWFLAKE_JWT authenticator", + "x-display": "textarea", + "x-secret": true, + }, + authenticator: { + type: "string", + title: "Authenticator", + description: "Override authenticator (e.g., SNOWFLAKE_JWT)", + "x-placeholder": "SNOWFLAKE_JWT", + }, + database: { + type: "string", + title: "Database", + description: "Snowflake database", + "x-placeholder": "your_database", + }, + schema: { + type: "string", + title: "Schema", + description: "Default schema", + "x-placeholder": "public", + }, + warehouse: { + type: "string", + title: "Warehouse", + description: "Compute warehouse", + "x-placeholder": "your_warehouse", + }, + role: { + type: "string", + title: "Role", + description: "Snowflake role", + "x-placeholder": "your_role", + }, + sql: { + type: "string", + title: "SQL", + description: "SQL query to run against your warehouse", + "x-display": "textarea", + "x-placeholder": "Input SQL", + "x-step": "explorer", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z0-9_]+$", + "x-placeholder": "my_model", + "x-step": "explorer", + }, + }, + required: ["sql", "name"], + allOf: [ + { + if: { properties: { connection_mode: { const: "dsn" } } }, + then: { required: ["dsn"] }, + else: { required: ["account", "user", "database", "warehouse"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/sqlite.ts b/web-common/src/features/templates/schemas/sqlite.ts new file mode 100644 index 00000000000..039f8550f83 --- /dev/null +++ b/web-common/src/features/templates/schemas/sqlite.ts @@ -0,0 +1,30 @@ +import type { MultiStepFormSchema } from "./types"; + +export const sqliteSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + db: { + type: "string", + title: "Database file", + description: "Path to SQLite db file", + "x-placeholder": "/path/to/sqlite.db", + "x-step": "source", + }, + table: { + type: "string", + title: "Table", + description: "SQLite table name", + "x-placeholder": "table", + "x-step": "source", + }, + name: { + type: "string", + title: "Source name", + description: "Name of the source", + "x-placeholder": "my_new_source", + "x-step": "source", + }, + }, + required: ["db", "table", "name"], +}; diff --git a/web-common/src/features/templates/schemas/types.ts b/web-common/src/features/templates/schemas/types.ts index d8ccd6b11b9..8d52bdadb1f 100644 --- a/web-common/src/features/templates/schemas/types.ts +++ b/web-common/src/features/templates/schemas/types.ts @@ -19,8 +19,8 @@ export type JSONSchemaField = { }; properties?: Record; required?: string[]; - "x-display"?: "radio" | "select" | "textarea" | "file"; - "x-step"?: "connector" | "source"; + "x-display"?: "radio" | "select" | "textarea" | "file" | "tabs"; + "x-step"?: "connector" | "source" | "explorer"; "x-secret"?: boolean; "x-visible-if"?: Record; "x-enum-labels"?: string[]; @@ -28,11 +28,18 @@ export type JSONSchemaField = { "x-placeholder"?: string; "x-hint"?: string; "x-accept"?: string; + "x-informational"?: boolean; + "x-docs-url"?: string; + "x-internal"?: boolean; /** * Explicit grouping for radio/select options: maps an option value to the * child field keys that should render beneath that option. */ "x-grouped-fields"?: Record; + /** + * Group fields under tab options for enum-driven tab layouts. + */ + "x-tab-group"?: Record; // Allow custom keywords such as errorMessage or future x-extensions. [key: string]: unknown; }; @@ -43,6 +50,9 @@ export type JSONSchemaCondition = { export type JSONSchemaConstraint = { required?: string[]; + properties?: Record; + // Allow custom keywords or overrides in constraints + [key: string]: unknown; }; export type JSONSchemaConditional = { @@ -59,6 +69,7 @@ export type JSONSchemaObject = { properties?: Record; required?: string[]; allOf?: JSONSchemaConditional[]; + oneOf?: JSONSchemaConstraint[]; }; export type MultiStepFormSchema = JSONSchemaObject; diff --git a/web-local/tests/connectors/multi-step-connector.spec.ts b/web-local/tests/connectors/multi-step-connector.spec.ts index 09e962d87b4..12659a17e93 100644 --- a/web-local/tests/connectors/multi-step-connector.spec.ts +++ b/web-local/tests/connectors/multi-step-connector.spec.ts @@ -222,4 +222,126 @@ test.describe("Multi-step connector wrapper", () => { const saveAnywayButton = page.getByRole("button", { name: "Save Anyway" }); await expect(saveAnywayButton).toBeHidden(); }); + + test("GCS connector - model form resets after first submission (HMAC)", async ({ + page, + }) => { + const hmacKey = process.env.RILL_RUNTIME_GCS_TEST_HMAC_KEY; + const hmacSecret = process.env.RILL_RUNTIME_GCS_TEST_HMAC_SECRET; + if (!hmacKey || !hmacSecret) { + test.skip( + true, + "RILL_RUNTIME_GCS_TEST_HMAC_KEY or RILL_RUNTIME_GCS_TEST_HMAC_SECRET is not set", + ); + } + test.slow(); + + const openGcsFlowWithHmac = async () => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + await page.getByRole("radio", { name: "HMAC keys" }).click(); + await page.getByRole("textbox", { name: "Access Key ID" }).fill(hmacKey!); + await page + .getByRole("textbox", { name: "Secret Access Key" }) + .fill(hmacSecret!); + const connectorCta = page.getByRole("button", { + name: "Test and Connect", + }); + await connectorCta.click(); + await expect(page.getByText("Model preview")).toBeVisible(); + }; + + // First submission attempt + await openGcsFlowWithHmac(); + const firstPath = + "gs://rilldata-public/github-analytics/Clickhouse/2025/06/commits_2025_06.parquet"; + const firstModelName = "gcs_model_one"; + await page.getByRole("textbox", { name: "GCS URI" }).fill(firstPath); + await page + .getByRole("textbox", { name: "Model name" }) + .fill(firstModelName); + + const submitCta = page.getByRole("button", { + name: "Import Data", + }); + await submitCta.click(); + + const dialog = page.getByRole("dialog"); + await dialog + .waitFor({ state: "detached", timeout: 10000 }) + .catch(async () => { + // If the modal is still open (e.g., reconciliation failed), close it and continue. + await page.keyboard.press("Escape"); + await dialog.waitFor({ state: "detached", timeout: 5000 }); + }); + + // Re-open and ensure model form is reset + await openGcsFlowWithHmac(); + await expect( + page.getByRole("textbox", { name: "GCS URI" }), + ).not.toHaveValue(firstPath); + await expect( + page.getByRole("textbox", { name: "Model name" }), + ).not.toHaveValue(firstModelName); + }); + + test("GCS connector - model YAML includes create_secrets_from_connectors", async ({ + page, + }) => { + const hmacKey = process.env.RILL_RUNTIME_GCS_TEST_HMAC_KEY; + const hmacSecret = process.env.RILL_RUNTIME_GCS_TEST_HMAC_SECRET; + if (!hmacKey || !hmacSecret) { + test.skip( + true, + "RILL_RUNTIME_GCS_TEST_HMAC_KEY or RILL_RUNTIME_GCS_TEST_HMAC_SECRET is not set", + ); + } + + const startGcsConnector = async () => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + await page.getByRole("radio", { name: "HMAC keys" }).click(); + await page.getByRole("textbox", { name: "Access Key ID" }).fill(hmacKey!); + await page + .getByRole("textbox", { name: "Secret Access Key" }) + .fill(hmacSecret!); + await page + .getByRole("dialog") + .getByRole("button", { name: "Test and Connect" }) + .click(); + await expect(page.getByText("Model preview")).toBeVisible(); + }; + + // Create first connector instance, then close modal + await startGcsConnector(); + await page.keyboard.press("Escape"); + await page.getByRole("dialog").waitFor({ state: "detached" }); + + // Create second connector instance and proceed to model import + await startGcsConnector(); + const modelName = "gcs_create_secrets_test"; + await page + .getByRole("textbox", { name: "GCS URI" }) + .fill( + "gs://rilldata-public/github-analytics/Clickhouse/2025/06/commits_2025_06.parquet", + ); + await page.getByRole("textbox", { name: "Model name" }).fill(modelName); + await page.getByRole("button", { name: "Import Data" }).click(); + + // Wait for navigation to the new model file + await page.waitForURL(`**/files/models/${modelName}.yaml`); + + // Verify YAML contains the connector reference and create_secrets_from_connectors with the second instance (gcs_1) + const codeEditor = page + .getByLabel("codemirror editor") + .getByRole("textbox"); + await expect(codeEditor).toContainText("connector: duckdb"); + await expect(codeEditor).toContainText( + /create_secrets_from_connectors:\s*\[gcs_1\]/, + ); + }); }); diff --git a/web-local/tests/connectors/save-anyway.spec.ts b/web-local/tests/connectors/save-anyway.spec.ts index 4fbb190ef5b..c3679f6f623 100644 --- a/web-local/tests/connectors/save-anyway.spec.ts +++ b/web-local/tests/connectors/save-anyway.spec.ts @@ -36,7 +36,8 @@ test.describe("Save Anyway feature", () => { // Wait for navigation to connector file, then for the editor to appear await expect(page).toHaveURL(/.*\/files\/connectors\/.*\.yaml/, { - timeout: 5000, + // Allow extra time for the file to be written and navigation to occur on slower CI + timeout: 6_000, }); const codeEditor = page .getByLabel("codemirror editor") diff --git a/web-local/tests/connectors/snowflake.spec.ts b/web-local/tests/connectors/snowflake.spec.ts new file mode 100644 index 00000000000..15de27c2946 --- /dev/null +++ b/web-local/tests/connectors/snowflake.spec.ts @@ -0,0 +1,47 @@ +import { expect } from "@playwright/test"; +import { test } from "../setup/base"; + +test.describe("Snowflake connector", () => { + test.use({ project: "Blank" }); + + test("submits connector with DSN", async ({ page }) => { + const dsn = process.env.RILL_RUNTIME_SNOWFLAKE_TEST_DSN; + if (!dsn) { + test.skip(true, "RILL_RUNTIME_SNOWFLAKE_TEST_DSN is not set"); + } + + // Open Add Data modal and pick Snowflake + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + await page.locator("#snowflake").click(); + await page.waitForSelector('form[id*="snowflake"]'); + + // Switch to the DSN tab + await page.getByRole("button", { name: "Enter connection string" }).click(); + + // Fill DSN field + await page + .getByRole("textbox", { name: "Snowflake Connection String" }) + .fill(dsn!); + + // Submit connector form + const submitButton = page + .getByRole("dialog") + .getByRole("button", { name: "Test and Connect" }); + await expect(submitButton).toBeEnabled(); + await submitButton.click(); + + // Expect navigation to the new connector file + await page.waitForURL("**/files/connectors/snowflake.yaml"); + + // Validate connector YAML contents + const codeEditor = page + .getByLabel("codemirror editor") + .getByRole("textbox"); + await expect(codeEditor).toContainText("type: connector"); + await expect(codeEditor).toContainText("driver: snowflake"); + await expect(codeEditor).toContainText( + 'dsn: "{{ .env.connector.snowflake.dsn }}"', + ); + }); +}); diff --git a/web-local/tests/rill-yaml.spec.ts b/web-local/tests/rill-yaml.spec.ts index 853cc2cfe7d..08d89b69b44 100644 --- a/web-local/tests/rill-yaml.spec.ts +++ b/web-local/tests/rill-yaml.spec.ts @@ -53,7 +53,7 @@ test.describe("Default olap_connector behavior", () => { await page.getByRole("button", { name: "Add Data" }).click(); await page.locator("#clickhouse").click(); - await page.locator("#managed").selectOption("rill-managed"); + await page.getByRole("radio", { name: "Rill-managed ClickHouse" }).check(); await page .getByRole("dialog", { name: "ClickHouse" }) .getByRole("button", {