From d4f4c6cf68910e1ef528ed1d6d93a78d9dea8160 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 10 Oct 2025 22:46:39 +0800 Subject: [PATCH 01/98] initial save anyway --- .../sources/modal/AddClickHouseForm.svelte | 9 +- .../features/sources/modal/AddDataForm.svelte | 126 +++++++++++++----- .../sources/modal/submitAddDataForm.ts | 62 +++++---- 3 files changed, 139 insertions(+), 58 deletions(-) diff --git a/web-common/src/features/sources/modal/AddClickHouseForm.svelte b/web-common/src/features/sources/modal/AddClickHouseForm.svelte index 9a2b764fbf7..600f27c2768 100644 --- a/web-common/src/features/sources/modal/AddClickHouseForm.svelte +++ b/web-common/src/features/sources/modal/AddClickHouseForm.svelte @@ -42,6 +42,9 @@ ) => void = () => {}; export let connectionTab: ConnectorType = "parameters"; export { paramsForm, dsnForm }; + export let isSavingAnyway: boolean = false; + + let saveAnyway = false; const dispatch = createEventDispatcher(); @@ -224,7 +227,7 @@ } try { - await submitAddConnectorForm(queryClient, connector, values); + await submitAddConnectorForm(queryClient, connector, values, saveAnyway); onClose(); } catch (e) { let error: string; @@ -261,6 +264,10 @@ dsnErrorDetails = details; setError(dsnError, dsnErrorDetails); } + } finally { + // Reset saveAnyway state after submission completes + saveAnyway = false; + isSavingAnyway = false; } } diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 71b196ea966..5e6a6d3978f 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -50,6 +50,9 @@ export let onBack: () => void; export let onClose: () => void; + let saveAnyway = false; + let isSavingAnyway = false; + const isSourceForm = formType === "source"; const isConnectorForm = formType === "connector"; @@ -132,6 +135,7 @@ let clickhouseConnectorType: ClickHouseConnectorType = "self-hosted"; let clickhouseParamsForm; let clickhouseDsnForm; + let clickhouseIsSavingAnyway: boolean; // Helper function to check if connector only has DSN (no tabs) function hasOnlyDsn() { @@ -333,9 +337,14 @@ try { if (formType === "source") { - await submitAddSourceForm(queryClient, connector, values); + await submitAddSourceForm(queryClient, connector, values, saveAnyway); } else { - await submitAddConnectorForm(queryClient, connector, values); + await submitAddConnectorForm( + queryClient, + connector, + values, + saveAnyway, + ); } onClose(); } catch (e) { @@ -375,6 +384,13 @@ paramsError = error; paramsErrorDetails = details; } + } finally { + // Reset saveAnyway state after submission completes + saveAnyway = false; + isSavingAnyway = false; + if (connector.name === "clickhouse") { + clickhouseIsSavingAnyway = false; + } } } @@ -408,6 +424,7 @@ bind:connectionTab bind:paramsForm={clickhouseParamsForm} bind:dsnForm={clickhouseDsnForm} + bind:isSavingAnyway={clickhouseIsSavingAnyway} on:submitting /> {:else if hasDsnFormOption} @@ -560,42 +577,85 @@ > - + {/if} + + + + diff --git a/web-common/src/features/sources/modal/submitAddDataForm.ts b/web-common/src/features/sources/modal/submitAddDataForm.ts index 2773c28822c..f24fca94d84 100644 --- a/web-common/src/features/sources/modal/submitAddDataForm.ts +++ b/web-common/src/features/sources/modal/submitAddDataForm.ts @@ -145,6 +145,7 @@ export async function submitAddSourceForm( queryClient: QueryClient, connector: V1ConnectorDriver, formValues: AddDataFormValues, + saveAnyway: boolean = false, ): Promise { const instanceId = get(runtime).instanceId; await beforeSubmitForm(instanceId, connector); @@ -196,17 +197,21 @@ export async function submitAddSourceForm( connector.name as string, ); } catch (error) { - // The source file was already created, so we need to delete it - await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); - const errorDetails = (error as any).details; - - throw { - message: error.message || "Unable to establish a connection", - details: - errorDetails && errorDetails !== error.message - ? errorDetails - : undefined, - }; + if (!saveAnyway) { + // The source file was already created, so we need to delete it + await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); + const errorDetails = (error as any).details; + + throw { + message: error.message || "Unable to establish a connection", + details: + errorDetails && errorDetails !== error.message + ? errorDetails + : undefined, + }; + } + // If saveAnyway is true, we continue despite the reconciliation error + // The file will be saved but may have connection issues } // Check for file errors @@ -216,10 +221,12 @@ export async function submitAddSourceForm( instanceId, newSourceFilePath, ); - if (errorMessage) { + if (errorMessage && !saveAnyway) { await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); throw new Error(errorMessage); } + // If saveAnyway is true, we continue despite file errors + // The file will be saved but may have parsing issues await goto(`/files/${newSourceFilePath}`); } @@ -228,6 +235,7 @@ export async function submitAddConnectorForm( queryClient: QueryClient, connector: V1ConnectorDriver, formValues: AddDataFormValues, + saveAnyway: boolean = false, ): Promise { const instanceId = get(runtime).instanceId; await beforeSubmitForm(instanceId, connector); @@ -286,17 +294,21 @@ export async function submitAddConnectorForm( connector.name as string, ); } catch (error) { - // The connector file was already created, so we need to delete it - await rollbackChanges(instanceId, newConnectorFilePath, originalEnvBlob); - const errorDetails = (error as any).details; - - throw { - message: error.message || "Unable to establish a connection", - details: - errorDetails && errorDetails !== error.message - ? errorDetails - : undefined, - }; + if (!saveAnyway) { + // The connector file was already created, so we need to delete it + await rollbackChanges(instanceId, newConnectorFilePath, originalEnvBlob); + const errorDetails = (error as any).details; + + throw { + message: error.message || "Unable to establish a connection", + details: + errorDetails && errorDetails !== error.message + ? errorDetails + : undefined, + }; + } + // If saveAnyway is true, we continue despite the reconciliation error + // The file will be saved but may have connection issues } // Check for file errors @@ -306,10 +318,12 @@ export async function submitAddConnectorForm( instanceId, newConnectorFilePath, ); - if (errorMessage) { + if (errorMessage && !saveAnyway) { await rollbackChanges(instanceId, newConnectorFilePath, originalEnvBlob); throw new Error(errorMessage); } + // If saveAnyway is true, we continue despite file errors + // The file will be saved but may have parsing issues if (OLAP_ENGINES.includes(connector.name as string)) { await setOlapConnectorInRillYAML(queryClient, instanceId, newConnectorName); From 1b9397e3e3261ee563f48b06b37fe46300a67054 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 15 Oct 2025 00:00:25 +0800 Subject: [PATCH 02/98] only one button shows a loading state at a time --- .../src/features/sources/modal/AddDataForm.svelte | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 5e6a6d3978f..fdf96abd257 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -585,8 +585,8 @@ ? clickhouseSubmitting : submitting} loading={connector.name === "clickhouse" - ? clickhouseSubmitting - : submitting} + ? clickhouseSubmitting && clickhouseIsSavingAnyway + : submitting && isSavingAnyway} loadingCopy="Saving..." onClick={() => { saveAnyway = true; @@ -621,8 +621,7 @@ clickhouseIsSavingAnyway : submitting || isSubmitDisabled || isSavingAnyway} loading={connector.name === "clickhouse" - ? clickhouseSubmitting && - !(isSavingAnyway || clickhouseIsSavingAnyway) + ? clickhouseSubmitting && !clickhouseIsSavingAnyway : submitting && !isSavingAnyway} loadingCopy={connector.name === "clickhouse" ? "Connecting..." @@ -633,12 +632,12 @@ > {#if connector.name === "clickhouse"} {#if clickhouseConnectorType === "rill-managed"} - {#if clickhouseSubmitting && !(isSavingAnyway || clickhouseIsSavingAnyway)} + {#if clickhouseSubmitting && !clickhouseIsSavingAnyway} Connecting... {:else} Connect {/if} - {:else if clickhouseSubmitting && !(isSavingAnyway || clickhouseIsSavingAnyway)} + {:else if clickhouseSubmitting && !clickhouseIsSavingAnyway} Testing connection... {:else} Test and Connect From b78dda2b0662e11b2129fad6e26cceff354e9ebd Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 15 Oct 2025 00:19:08 +0800 Subject: [PATCH 03/98] just save the files --- .../sources/modal/submitAddDataForm.ts | 142 ++++++++++-------- 1 file changed, 76 insertions(+), 66 deletions(-) diff --git a/web-common/src/features/sources/modal/submitAddDataForm.ts b/web-common/src/features/sources/modal/submitAddDataForm.ts index f24fca94d84..e009fb1a9d5 100644 --- a/web-common/src/features/sources/modal/submitAddDataForm.ts +++ b/web-common/src/features/sources/modal/submitAddDataForm.ts @@ -179,25 +179,34 @@ export async function submitAddSourceForm( "source", ); - // Make sure the file has reconciled before testing the connection - await runtimeServicePutFileAndWaitForReconciliation(instanceId, { - path: ".env", - blob: newEnvBlob, - create: true, - createOnly: false, - }); + if (saveAnyway) { + // When saving anyway, just create the .env file without waiting for reconciliation + await runtimeServicePutFile(instanceId, { + path: ".env", + blob: newEnvBlob, + create: true, + createOnly: false, + }); + // Skip reconciliation and error checking - just save the files + } else { + // Make sure the file has reconciled before testing the connection + await runtimeServicePutFileAndWaitForReconciliation(instanceId, { + path: ".env", + blob: newEnvBlob, + create: true, + createOnly: false, + }); - // Wait for source resource-level reconciliation - // This must happen after .env reconciliation since sources depend on secrets - try { - await waitForResourceReconciliation( - instanceId, - newSourceName, - ResourceKind.Model, - connector.name as string, - ); - } catch (error) { - if (!saveAnyway) { + // Wait for source resource-level reconciliation + // This must happen after .env reconciliation since sources depend on secrets + try { + await waitForResourceReconciliation( + instanceId, + newSourceName, + ResourceKind.Model, + connector.name as string, + ); + } catch (error) { // The source file was already created, so we need to delete it await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); const errorDetails = (error as any).details; @@ -210,23 +219,19 @@ export async function submitAddSourceForm( : undefined, }; } - // If saveAnyway is true, we continue despite the reconciliation error - // The file will be saved but may have connection issues - } - // Check for file errors - // If the model file has errors, rollback the changes - const errorMessage = await fileArtifacts.checkFileErrors( - queryClient, - instanceId, - newSourceFilePath, - ); - if (errorMessage && !saveAnyway) { - await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); - throw new Error(errorMessage); + // Check for file errors + // If the model file has errors, rollback the changes + const errorMessage = await fileArtifacts.checkFileErrors( + queryClient, + instanceId, + newSourceFilePath, + ); + if (errorMessage) { + await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); + throw new Error(errorMessage); + } } - // If saveAnyway is true, we continue despite file errors - // The file will be saved but may have parsing issues await goto(`/files/${newSourceFilePath}`); } @@ -276,25 +281,34 @@ export async function submitAddConnectorForm( newConnectorName, ); - // Make sure the file has reconciled before testing the connection - await runtimeServicePutFileAndWaitForReconciliation(instanceId, { - path: ".env", - blob: newEnvBlob, - create: true, - createOnly: false, - }); + if (saveAnyway) { + // When saving anyway, just create the .env file without waiting for reconciliation + await runtimeServicePutFile(instanceId, { + path: ".env", + blob: newEnvBlob, + create: true, + createOnly: false, + }); + // Skip reconciliation and error checking - just save the files + } else { + // Make sure the file has reconciled before testing the connection + await runtimeServicePutFileAndWaitForReconciliation(instanceId, { + path: ".env", + blob: newEnvBlob, + create: true, + createOnly: false, + }); - // Wait for connector resource-level reconciliation - // This must happen after .env reconciliation since connectors depend on secrets - try { - await waitForResourceReconciliation( - instanceId, - newConnectorName, - ResourceKind.Connector, - connector.name as string, - ); - } catch (error) { - if (!saveAnyway) { + // Wait for connector resource-level reconciliation + // This must happen after .env reconciliation since connectors depend on secrets + try { + await waitForResourceReconciliation( + instanceId, + newConnectorName, + ResourceKind.Connector, + connector.name as string, + ); + } catch (error) { // The connector file was already created, so we need to delete it await rollbackChanges(instanceId, newConnectorFilePath, originalEnvBlob); const errorDetails = (error as any).details; @@ -307,23 +321,19 @@ export async function submitAddConnectorForm( : undefined, }; } - // If saveAnyway is true, we continue despite the reconciliation error - // The file will be saved but may have connection issues - } - // Check for file errors - // If the connector file has errors, rollback the changes - const errorMessage = await fileArtifacts.checkFileErrors( - queryClient, - instanceId, - newConnectorFilePath, - ); - if (errorMessage && !saveAnyway) { - await rollbackChanges(instanceId, newConnectorFilePath, originalEnvBlob); - throw new Error(errorMessage); + // Check for file errors + // If the connector file has errors, rollback the changes + const errorMessage = await fileArtifacts.checkFileErrors( + queryClient, + instanceId, + newConnectorFilePath, + ); + if (errorMessage) { + await rollbackChanges(instanceId, newConnectorFilePath, originalEnvBlob); + throw new Error(errorMessage); + } } - // If saveAnyway is true, we continue despite file errors - // The file will be saved but may have parsing issues if (OLAP_ENGINES.includes(connector.name as string)) { await setOlapConnectorInRillYAML(queryClient, instanceId, newConnectorName); From 1ad35caeb32e9e98b278bb8e545211923894efb8 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 15 Oct 2025 23:18:04 +0800 Subject: [PATCH 04/98] wip --- .../sources/modal/AddClickHouseForm.svelte | 22 +++++++++++++++++-- .../features/sources/modal/AddDataForm.svelte | 14 ++++++++---- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/web-common/src/features/sources/modal/AddClickHouseForm.svelte b/web-common/src/features/sources/modal/AddClickHouseForm.svelte index 600f27c2768..6eb028c65c1 100644 --- a/web-common/src/features/sources/modal/AddClickHouseForm.svelte +++ b/web-common/src/features/sources/modal/AddClickHouseForm.svelte @@ -46,6 +46,9 @@ let saveAnyway = false; + // Sync the local saveAnyway with the parent's isSavingAnyway state + $: saveAnyway = isSavingAnyway; + const dispatch = createEventDispatcher(); // ClickHouse schema includes the 'managed' property for backend compatibility @@ -213,7 +216,17 @@ { type: "success" | "failure" } >; }) { - if (!event.form.valid) return; + console.log("handleOnUpdate called:", { + formValid: event.form.valid, + saveAnyway, + connectionTab, + formData: event.form.data, + }); + + if (!event.form.valid && !saveAnyway) { + console.log("Form is invalid and saveAnyway is false, returning early"); + return; + } const values = { ...event.form.data }; // Ensure ClickHouse Cloud specific requirements are met @@ -227,6 +240,11 @@ } try { + console.log("Calling submitAddConnectorForm with:", { + connector: connector.name, + values, + saveAnyway, + }); await submitAddConnectorForm(queryClient, connector, values, saveAnyway); onClose(); } catch (e) { @@ -266,7 +284,7 @@ } } finally { // Reset saveAnyway state after submission completes - saveAnyway = false; + // Note: saveAnyway is now reactive and will be reset when isSavingAnyway changes isSavingAnyway = false; } } diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index fdf96abd257..e6620756d7c 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -332,7 +332,7 @@ cancel: () => void; result: Extract; }) { - if (!event.form.valid) return; + if (!event.form.valid && !saveAnyway) return; const values = event.form.data; try { @@ -593,7 +593,7 @@ isSavingAnyway = true; // For ClickHouse, also set the flag in the child component if (connector.name === "clickhouse") { - // The binding will sync this automatically + clickhouseIsSavingAnyway = true; } // Trigger form submission by dispatching a submit event const formElement = document.getElementById( @@ -624,7 +624,9 @@ ? clickhouseSubmitting && !clickhouseIsSavingAnyway : submitting && !isSavingAnyway} loadingCopy={connector.name === "clickhouse" - ? "Connecting..." + ? clickhouseIsSavingAnyway + ? "Saving..." + : "Connecting..." : "Testing connection..."} form={connector.name === "clickhouse" ? clickhouseFormId : formId} submitForm @@ -632,11 +634,15 @@ > {#if connector.name === "clickhouse"} {#if clickhouseConnectorType === "rill-managed"} - {#if clickhouseSubmitting && !clickhouseIsSavingAnyway} + {#if clickhouseSubmitting && clickhouseIsSavingAnyway} + Saving... + {:else if clickhouseSubmitting && !clickhouseIsSavingAnyway} Connecting... {:else} Connect {/if} + {:else if clickhouseSubmitting && clickhouseIsSavingAnyway} + Saving... {:else if clickhouseSubmitting && !clickhouseIsSavingAnyway} Testing connection... {:else} From 41f3bda04ef2b679e47c1e02e7905d41263e03c2 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 16 Oct 2025 00:24:24 +0800 Subject: [PATCH 05/98] show save anyway right away --- .../features/sources/modal/AddDataForm.svelte | 105 ++++++++++++++---- 1 file changed, 81 insertions(+), 24 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index e6620756d7c..48aa7ae8d96 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -52,6 +52,7 @@ let saveAnyway = false; let isSavingAnyway = false; + let showSaveAnyway = false; const isSourceForm = formType === "source"; const isConnectorForm = formType === "connector"; @@ -136,6 +137,7 @@ let clickhouseParamsForm; let clickhouseDsnForm; let clickhouseIsSavingAnyway: boolean; + let clickhouseShowSaveAnyway: boolean = false; // Helper function to check if connector only has DSN (no tabs) function hasOnlyDsn() { @@ -212,6 +214,77 @@ // Emit the submitting state to the parent $: dispatch("submitting", { submitting }); + async function handleSaveAnyway() { + saveAnyway = true; + isSavingAnyway = true; + // For ClickHouse, also set the flag in the child component + if (connector.name === "clickhouse") { + clickhouseIsSavingAnyway = true; + } + + // Get the current form values based on the active form + let values: Record; + if (connector.name === "clickhouse") { + values = + connectionTab === "dsn" ? $clickhouseDsnForm : $clickhouseParamsForm; + } else { + values = hasOnlyDsn() || connectionTab === "dsn" ? $dsnForm : $paramsForm; + } + + try { + if (formType === "source") { + await submitAddSourceForm(queryClient, connector, values, true); + } else { + await submitAddConnectorForm(queryClient, connector, values, true); + } + onClose(); + } catch (e) { + let error: string; + let details: string | undefined = undefined; + + // Handle different error types + if (e instanceof Error) { + error = e.message; + details = undefined; + } else if (e?.message && e?.details) { + error = e.message; + details = e.details !== e.message ? e.details : undefined; + } else if (e?.response?.data) { + const originalMessage = e.response.data.message; + const humanReadable = humanReadableErrorMessage( + connector.name, + e.response.data.code, + originalMessage, + ); + error = humanReadable; + details = + humanReadable !== originalMessage ? originalMessage : undefined; + } else if (e?.message) { + error = e.message; + details = undefined; + } else { + error = "Unknown error"; + details = undefined; + } + + // Keep error state for each form - match the display logic + if (hasOnlyDsn() || connectionTab === "dsn") { + dsnError = error; + dsnErrorDetails = details; + } else { + paramsError = error; + paramsErrorDetails = details; + } + } finally { + // Reset saveAnyway state after submission completes + saveAnyway = false; + isSavingAnyway = false; + if (connector.name === "clickhouse") { + clickhouseIsSavingAnyway = false; + } + } + } + function getClickHouseYamlPreview( values: Record, connectorType: ClickHouseConnectorType, @@ -332,6 +405,9 @@ cancel: () => void; result: Extract; }) { + // Show Save Anyway button as soon as form submission starts + showSaveAnyway = true; + if (!event.form.valid && !saveAnyway) return; const values = event.form.data; @@ -425,6 +501,7 @@ bind:paramsForm={clickhouseParamsForm} bind:dsnForm={clickhouseDsnForm} bind:isSavingAnyway={clickhouseIsSavingAnyway} + bind:showSaveAnyway={clickhouseShowSaveAnyway} on:submitting /> {:else if hasDsnFormOption} @@ -578,35 +655,15 @@
- - {#if dsnError || paramsError || clickhouseError} + + {#if dsnError || paramsError || clickhouseError || showSaveAnyway || clickhouseShowSaveAnyway}
- - {#if dsnError || paramsError || clickhouseError || showSaveAnyway || clickhouseShowSaveAnyway} + + {#if showSaveAnyway || clickhouseShowSaveAnyway}
- - {#if showSaveAnyway || clickhouseShowSaveAnyway} + + {#if isConnectorForm && (showSaveAnyway || clickhouseShowSaveAnyway)}
- - {#if isConnectorForm && (showSaveAnyway || clickhouseShowSaveAnyway)} + {#if shouldShowSaveAnywayButton}
From a171bd234fc7fbbdef0d5b977dffad2269be45c4 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 28 Oct 2025 17:46:24 +0800 Subject: [PATCH 22/98] track file paths to avoid getting auto removed --- .../sources/modal/submitAddDataForm.ts | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/web-common/src/features/sources/modal/submitAddDataForm.ts b/web-common/src/features/sources/modal/submitAddDataForm.ts index a5488943715..422a88babc2 100644 --- a/web-common/src/features/sources/modal/submitAddDataForm.ts +++ b/web-common/src/features/sources/modal/submitAddDataForm.ts @@ -41,6 +41,10 @@ interface AddDataFormValues { [key: string]: unknown; } +// Track connector file paths that were created via Save Anyway so +// in-flight Test-and-Connect submissions don't roll them back. +const savedAnywayPaths = new Set(); + async function beforeSubmitForm( instanceId: string, connector?: V1ConnectorDriver, @@ -247,6 +251,9 @@ async function saveConnectorAnyway( EntityType.Connector, ); + // Mark to avoid rollback by concurrent submissions + savedAnywayPaths.add(newConnectorFilePath); + // Always create/overwrite to ensure the file is created immediately await runtimeServicePutFile(resolvedInstanceId, { path: newConnectorFilePath, @@ -413,12 +420,15 @@ export async function submitAddConnectorForm( connector.name as string, ); } catch (error) { - // The connector file was already created, so we need to delete it - await rollbackChanges( - instanceId, - newConnectorFilePath, - originalEnvBlob, - ); + // The connector file was already created, so we would delete it + // unless Save Anyway has already created it intentionally. + if (!savedAnywayPaths.has(newConnectorFilePath)) { + await rollbackChanges( + instanceId, + newConnectorFilePath, + originalEnvBlob, + ); + } const errorDetails = (error as any).details; throw { @@ -438,11 +448,13 @@ export async function submitAddConnectorForm( newConnectorFilePath, ); if (errorMessage) { - await rollbackChanges( - instanceId, - newConnectorFilePath, - originalEnvBlob, - ); + if (!savedAnywayPaths.has(newConnectorFilePath)) { + await rollbackChanges( + instanceId, + newConnectorFilePath, + originalEnvBlob, + ); + } throw new Error(errorMessage); } } From 3cadcf1faecc2c0c90030f2b307f8c5cc70cd4ec Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 28 Oct 2025 17:48:28 +0800 Subject: [PATCH 23/98] fix during issue 2 --- .../src/features/sources/modal/AddDataForm.svelte | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 584a3390a9f..80acb42aba7 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -165,10 +165,12 @@ if (hasOnlyDsn() || connectionTab === "dsn") { // DSN form: check required DSN properties for (const property of dsnProperties) { - if (property.required) { - const key = String(property.key); - const value = $dsnForm[key]; - if (isEmpty(value) || $dsnErrors[key]?.length) return true; + const key = String(property.key); + const value = $dsnForm[key]; + // DSN should be present even if not marked required in metadata + const mustBePresent = property.required || key === "dsn"; + if (mustBePresent && (isEmpty(value) || $dsnErrors[key]?.length)) { + return true; } } return false; From a39f78901d122d6d8df2ab4b5a9ba9d779c15f1d Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 31 Oct 2025 12:51:27 +0800 Subject: [PATCH 24/98] colocate connectors --- web-local/tests/{ => connectors}/bigquery-connector.spec.ts | 0 web-local/tests/{ => connectors}/olap-connectors.spec.ts | 4 ++-- web-local/tests/{ => connectors}/save-anyway.spec.ts | 0 web-local/tests/{ => connectors}/test-connection.spec.ts | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename web-local/tests/{ => connectors}/bigquery-connector.spec.ts (100%) rename web-local/tests/{ => connectors}/olap-connectors.spec.ts (98%) rename web-local/tests/{ => connectors}/save-anyway.spec.ts (100%) rename web-local/tests/{ => connectors}/test-connection.spec.ts (99%) diff --git a/web-local/tests/bigquery-connector.spec.ts b/web-local/tests/connectors/bigquery-connector.spec.ts similarity index 100% rename from web-local/tests/bigquery-connector.spec.ts rename to web-local/tests/connectors/bigquery-connector.spec.ts diff --git a/web-local/tests/olap-connectors.spec.ts b/web-local/tests/connectors/olap-connectors.spec.ts similarity index 98% rename from web-local/tests/olap-connectors.spec.ts rename to web-local/tests/connectors/olap-connectors.spec.ts index 3a953d44165..d6d40581591 100644 --- a/web-local/tests/olap-connectors.spec.ts +++ b/web-local/tests/connectors/olap-connectors.spec.ts @@ -1,6 +1,6 @@ import { expect } from "@playwright/test"; -import { test } from "./setup/base"; -import { ClickHouseTestContainer } from "./utils/clickhouse"; +import { test } from "../setup/base"; +import { ClickHouseTestContainer } from "../utils/clickhouse"; test.describe("ClickHouse connector", () => { /* diff --git a/web-local/tests/save-anyway.spec.ts b/web-local/tests/connectors/save-anyway.spec.ts similarity index 100% rename from web-local/tests/save-anyway.spec.ts rename to web-local/tests/connectors/save-anyway.spec.ts diff --git a/web-local/tests/test-connection.spec.ts b/web-local/tests/connectors/test-connection.spec.ts similarity index 99% rename from web-local/tests/test-connection.spec.ts rename to web-local/tests/connectors/test-connection.spec.ts index a074002544a..e571c62d3bf 100644 --- a/web-local/tests/test-connection.spec.ts +++ b/web-local/tests/connectors/test-connection.spec.ts @@ -1,5 +1,5 @@ import { expect } from "@playwright/test"; -import { test } from "./setup/base"; +import { test } from "../setup/base"; test.describe("Test Connection", () => { test.use({ project: "Blank" }); From b5b5489de325a16f1a93ced911a51bfcfd80f592 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 31 Oct 2025 19:10:39 +0800 Subject: [PATCH 25/98] lint --- .../features/sources/modal/AddDataForm.svelte | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 2508a2050b2..d0f68faa388 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -445,44 +445,10 @@ $: shouldShowSaveAnywayButton = isConnectorForm && (showSaveAnyway || clickhouseShowSaveAnyway); - $: primaryDisabled = isClickhouse - ? clickhouseSubmitting || clickhouseIsSubmitDisabled || saveAnyway - : submitting || isSubmitDisabled || saveAnyway; - - $: primaryLoading = isClickhouse - ? clickhouseSubmitting && !saveAnyway - : submitting && !saveAnyway; - $: saveAnywayLoading = isClickhouse ? clickhouseSubmitting && saveAnyway : submitting && saveAnyway; - $: primaryLoadingCopy = isClickhouse - ? saveAnyway - ? "Saving..." - : "Connecting..." - : "Testing connection..."; - - $: primaryLabel = (() => { - if (isClickhouse) { - if (clickhouseConnectorType === "rill-managed") { - if (clickhouseSubmitting && saveAnyway) return "Saving..."; - if (clickhouseSubmitting && !saveAnyway) return "Connecting..."; - return "Connect"; - } else { - if (clickhouseSubmitting && saveAnyway) return "Saving..."; - if (clickhouseSubmitting && !saveAnyway) return "Testing connection..."; - return "Test and Connect"; - } - } else if (isConnectorForm) { - if (submitting && !saveAnyway) return "Testing connection..."; - return "Test and Connect"; - } else { - if (submitting && !saveAnyway) return "Testing connection..."; - return "Test and Add data"; - } - })(); - function onStringInputChange(event: Event) { const target = event.target as HTMLInputElement; const { name, value } = target; From 3bc5a81d9d316571235016bab6d15b3a0c7d3bca Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 31 Oct 2025 19:18:34 +0800 Subject: [PATCH 26/98] prettier --- web-common/src/features/sources/modal/submitAddDataForm.ts | 1 - web-local/tests/connectors/bigquery-connector.spec.ts | 2 +- web-local/tests/connectors/save-anyway.spec.ts | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/web-common/src/features/sources/modal/submitAddDataForm.ts b/web-common/src/features/sources/modal/submitAddDataForm.ts index 8bda2e66906..ce2f1e4ba50 100644 --- a/web-common/src/features/sources/modal/submitAddDataForm.ts +++ b/web-common/src/features/sources/modal/submitAddDataForm.ts @@ -416,7 +416,6 @@ export async function submitAddConnectorForm( instanceId, newConnectorName, ResourceKind.Connector, - connector.name as string, ); } catch (error) { // The connector file was already created, so we would delete it diff --git a/web-local/tests/connectors/bigquery-connector.spec.ts b/web-local/tests/connectors/bigquery-connector.spec.ts index d447b6ddda2..4e35063a8e4 100644 --- a/web-local/tests/connectors/bigquery-connector.spec.ts +++ b/web-local/tests/connectors/bigquery-connector.spec.ts @@ -1,5 +1,5 @@ import { expect } from "@playwright/test"; -import { test } from "./setup/base"; +import { test } from "../setup/base"; import * as path from "path"; import { fileURLToPath } from "url"; import { writeFileSync, unlinkSync, existsSync } from "fs"; diff --git a/web-local/tests/connectors/save-anyway.spec.ts b/web-local/tests/connectors/save-anyway.spec.ts index d2dc7cd7e26..f56c325c702 100644 --- a/web-local/tests/connectors/save-anyway.spec.ts +++ b/web-local/tests/connectors/save-anyway.spec.ts @@ -1,5 +1,5 @@ import { expect } from "@playwright/test"; -import { test } from "./setup/base"; +import { test } from "../setup/base"; test.describe("Save Anyway feature", () => { test.use({ project: "Blank" }); From 1761ee97b8f995da1e6aaf5af5fa331e9a03ce97 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 31 Oct 2025 23:21:12 +0800 Subject: [PATCH 27/98] initial class manager --- .../features/sources/modal/AddDataForm.svelte | 87 +++++---------- .../sources/modal/AddDataFormManager.ts | 103 ++++++++++++++++++ 2 files changed, 129 insertions(+), 61 deletions(-) create mode 100644 web-common/src/features/sources/modal/AddDataFormManager.ts diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index d0f68faa388..7fddf90fca5 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -6,12 +6,7 @@ import { type V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; import type { ActionResult } from "@sveltejs/kit"; import { createEventDispatcher } from "svelte"; - import { - defaults, - superForm, - type SuperValidated, - } from "sveltekit-superforms"; - import { yup } from "sveltekit-superforms/adapters"; + import type { SuperValidated } from "sveltekit-superforms"; import { inferSourceName, prepareSourceFormData, @@ -23,10 +18,6 @@ submitAddSourceForm, } from "./submitAddDataForm"; import type { AddDataFormType, ConnectorType } from "./types"; - import { - dsnSchema as dsnValidation, - getValidationSchemaForConnector, - } from "./FormValidation"; import AddClickHouseForm from "./AddClickHouseForm.svelte"; import NeedHelpText from "./NeedHelpText.svelte"; import Tabs from "@rilldata/web-common/components/forms/Tabs.svelte"; @@ -47,6 +38,7 @@ import FormRenderer from "./FormRenderer.svelte"; import YamlPreview from "./YamlPreview.svelte"; import GCSMultiStepForm from "./GCSMultiStepForm.svelte"; + import { AddDataFormManager } from "./AddDataFormManager"; const dispatch = createEventDispatcher(); @@ -63,6 +55,14 @@ let connectionTab: ConnectorType = "parameters"; + // Form Generation Manager (phase 1) + const formManager = new AddDataFormManager({ + connector, + formType, + onParamsUpdate: handleOnUpdate, + onDsnUpdate: handleOnUpdate, + }); + // Simple multi-step state management const isMultiStepConnector = MULTI_STEP_CONNECTORS.includes( connector.name ?? "", @@ -95,34 +95,12 @@ $: effectiveFormType = isMultiStepConnector && stepState.step === "source" ? "source" : formType; - $: formHeight = ["clickhouse", "snowflake", "salesforce"].includes( - connector.name ?? "", - ) - ? "max-h-[38.5rem] min-h-[38.5rem]" - : "max-h-[34.5rem] min-h-[34.5rem]"; + $: formHeight = formManager.formHeight; // Form 1: Individual parameters - const paramsFormId = `add-data-${connector.name}-form`; - const properties = - (isSourceForm - ? connector.sourceProperties - : connector.configProperties?.filter( - (property) => property.key !== "dsn", - )) ?? []; - - // Filter properties based on connector type - const filteredParamsProperties = (() => { - // FIXME: https://linear.app/rilldata/issue/APP-408/support-ducklake-in-the-ui - if (connector.name === "duckdb") { - return properties.filter( - (property) => property.key !== "attach" && property.key !== "mode", - ); - } - // For other connectors, filter out noPrompt properties - return properties.filter((property) => !property.noPrompt); - })(); - const schema = yup(getValidationSchemaForConnector(connector.name as string)); - const initialFormValues = getInitialFormValuesFromProperties(properties); + const paramsFormId = formManager.paramsFormId; + const properties = formManager.properties; + const filteredParamsProperties = formManager.filteredParamsProperties; const { form: paramsForm, errors: paramsErrors, @@ -130,28 +108,16 @@ tainted: paramsTainted, submit: paramsSubmit, submitting: paramsSubmitting, - } = superForm(initialFormValues, { - SPA: true, - validators: schema, - onUpdate: handleOnUpdate, - resetForm: false, - }); + } = formManager.params; let paramsError: string | null = null; let paramsErrorDetails: string | undefined = undefined; // Form 2: DSN // SuperForms are not meant to have dynamic schemas, so we use a different form instance for the DSN form - const hasDsnFormOption = - isConnectorForm && - connector.configProperties?.some((property) => property.key === "dsn") && - connector.configProperties?.some((property) => property.key !== "dsn"); - const dsnFormId = `add-data-${connector.name}-dsn-form`; - const dsnProperties = - connector.configProperties?.filter((property) => property.key === "dsn") ?? - []; - - const filteredDsnProperties = dsnProperties; - const dsnYupSchema = yup(dsnValidation); + const hasDsnFormOption = formManager.hasDsnFormOption; + const dsnFormId = formManager.dsnFormId; + const dsnProperties = formManager.dsnProperties; + const filteredDsnProperties = formManager.filteredDsnProperties; const { form: dsnForm, errors: dsnErrors, @@ -159,12 +125,7 @@ tainted: dsnTainted, submit: dsnSubmit, submitting: dsnSubmitting, - } = superForm(defaults(dsnYupSchema), { - SPA: true, - validators: dsnYupSchema, - onUpdate: handleOnUpdate, - resetForm: false, - }); + } = formManager.dsn; let dsnError: string | null = null; let dsnErrorDetails: string | undefined = undefined; @@ -210,7 +171,10 @@ const value = $dsnForm[key]; // DSN should be present even if not marked required in metadata const mustBePresent = property.required || key === "dsn"; - if (mustBePresent && (isEmpty(value) || $dsnErrors[key]?.length)) { + if ( + mustBePresent && + (isEmpty(value) || /**/ ($dsnErrors[key] as any)?.length) + ) { return true; } } @@ -228,7 +192,8 @@ const value = $paramsForm[key]; // Normal validation for all properties - if (isEmpty(value) || $paramsErrors[key]?.length) return true; + if (isEmpty(value) || /**/ ($paramsErrors[key] as any)?.length) + return true; } } return false; diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts new file mode 100644 index 00000000000..09ef5a14cb0 --- /dev/null +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -0,0 +1,103 @@ +import { superForm, defaults } from "sveltekit-superforms"; +import { yup } from "sveltekit-superforms/adapters"; +import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; +import type { AddDataFormType } from "./types"; +import { getValidationSchemaForConnector, dsnSchema } from "./FormValidation"; +import { getInitialFormValuesFromProperties } from "../sourceUtils"; + +export class AddDataFormManager { + formHeight: string; + paramsFormId: string; + dsnFormId: string; + hasDsnFormOption: boolean; + hasOnlyDsn: boolean; + properties: any[]; + filteredParamsProperties: any[]; + dsnProperties: any[]; + filteredDsnProperties: any[]; + + // superforms instances + params: ReturnType; + dsn: ReturnType; + + constructor(args: { + connector: V1ConnectorDriver; + formType: AddDataFormType; + onParamsUpdate: any; + onDsnUpdate: any; + }) { + const { connector, formType, onParamsUpdate, onDsnUpdate } = args; + + // Layout height + this.formHeight = ["clickhouse", "snowflake", "salesforce"].includes( + connector.name ?? "", + ) + ? "max-h-[38.5rem] min-h-[38.5rem]" + : "max-h-[34.5rem] min-h-[34.5rem]"; + + // IDs + this.paramsFormId = `add-data-${connector.name}-form`; + this.dsnFormId = `add-data-${connector.name}-dsn-form`; + + const isSourceForm = formType === "source"; + const isConnectorForm = formType === "connector"; + + // Base properties + this.properties = + (isSourceForm + ? connector.sourceProperties + : connector.configProperties?.filter((p) => p.key !== "dsn")) ?? []; + + // Filter properties based on connector type + this.filteredParamsProperties = (() => { + if (connector.name === "duckdb") { + return this.properties.filter( + (p) => p.key !== "attach" && p.key !== "mode", + ); + } + return this.properties.filter((p) => !p.noPrompt); + })(); + + // DSN properties + this.dsnProperties = + connector.configProperties?.filter((p) => p.key === "dsn") ?? []; + this.filteredDsnProperties = this.dsnProperties; + + // DSN flags + this.hasDsnFormOption = !!( + isConnectorForm && + connector.configProperties?.some((p) => p.key === "dsn") && + connector.configProperties?.some((p) => p.key !== "dsn") + ); + this.hasOnlyDsn = !!( + isConnectorForm && + connector.configProperties?.some((p) => p.key === "dsn") && + !connector.configProperties?.some((p) => p.key !== "dsn") + ); + + // Superforms: params + const schema = yup( + getValidationSchemaForConnector(connector.name as string), + ); + const initialFormValues = getInitialFormValuesFromProperties( + this.properties, + ); + this.params = superForm(initialFormValues, { + SPA: true, + validators: schema, + onUpdate: onParamsUpdate, + resetForm: false, + } as any); + + // Superforms: dsn + const dsnYupSchema = yup(dsnSchema); + this.dsn = superForm(defaults(dsnYupSchema), { + SPA: true, + validators: dsnYupSchema, + onUpdate: onDsnUpdate, + resetForm: false, + } as any); + } + + destroy() {} +} From 267e850606c09a3225a1832021d84bbc8ff86a62 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 31 Oct 2025 23:52:36 +0800 Subject: [PATCH 28/98] move business logic methods --- .../features/sources/modal/AddDataForm.svelte | 143 ++++-------------- .../sources/modal/AddDataFormManager.ts | 123 ++++++++++++++- 2 files changed, 151 insertions(+), 115 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 7fddf90fca5..ad882b1825c 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -55,12 +55,24 @@ let connectionTab: ConnectorType = "parameters"; + // Wire manager-provided onUpdate after declaration below + let handleOnUpdate: < + T extends Record, + M = any, + In extends Record = T, + >(event: { + form: SuperValidated; + formEl: HTMLFormElement; + cancel: () => void; + result: Extract; + }) => Promise; + // Form Generation Manager (phase 1) const formManager = new AddDataFormManager({ connector, formType, - onParamsUpdate: handleOnUpdate, - onDsnUpdate: handleOnUpdate, + onParamsUpdate: (e: any) => handleOnUpdate(e), + onDsnUpdate: (e: any) => handleOnUpdate(e), }); // Simple multi-step state management @@ -415,123 +427,26 @@ : submitting && saveAnyway; function onStringInputChange(event: Event) { - const target = event.target as HTMLInputElement; - const { name, value } = target; - - if (name === "path") { - if ($paramsTainted?.name) return; - const name = inferSourceName(connector, value); - if (name) - paramsForm.update( - ($form) => { - $form.name = name; - return $form; - }, - { taint: false }, - ); - } + formManager.onStringInputChange(event); } - async function handleOnUpdate< - T extends Record, - M = any, - In extends Record = T, - >(event: { - form: SuperValidated; - formEl: HTMLFormElement; - cancel: () => void; - result: Extract; - }) { - // Show Save Anyway button as soon as form submission starts - only for connector forms - if (isConnectorForm) { - showSaveAnyway = true; - } - - if (!event.form.valid && !saveAnyway) return; - - const values = event.form.data; - - try { - // Apply ClickHouse Cloud requirements if needed - let processedValues = applyClickHouseCloudRequirements(values); - - if (isMultiStepConnector && stepState.step === "source") { - // Step 2: Create source with stored connector config - await submitAddSourceForm(queryClient, connector, processedValues); - onClose(); - } else if (isMultiStepConnector && stepState.step === "connector") { - // Step 1: Create connector and transition to step 2 - await submitAddConnectorForm( - queryClient, - connector, - processedValues, - true, - ); - setConnectorConfig(processedValues); - setStep("source"); - return; // Don't close the modal, just transition to step 2 - } else if (effectiveFormType === "source") { - // Regular source form - await submitAddSourceForm(queryClient, connector, processedValues); - onClose(); - } else { - // Regular connector form - await submitAddConnectorForm( - queryClient, - connector, - processedValues, - true, - ); - onClose(); - } - } catch (e) { - const { message, details } = normalizeConnectorError( - connector.name ?? "", - e, - ); - - // Keep error state for each form - match the display logic - if (hasOnlyDsn() || connectionTab === "dsn") { - dsnError = message; - dsnErrorDetails = details; - } else { - paramsError = message; - paramsErrorDetails = details; - } - } finally { - // Reset saveAnyway state after submission completes - saveAnyway = false; - } - } + handleOnUpdate = formManager.makeOnUpdate({ + onClose, + queryClient, + getConnectionTab: () => connectionTab, + setParamsError: (message: string | null, details?: string) => { + paramsError = message; + paramsErrorDetails = details; + }, + setDsnError: (message: string | null, details?: string) => { + dsnError = message; + dsnErrorDetails = details; + }, + }); // Handle file upload for credential files async function handleFileUpload(file: File): Promise { - try { - const content = await file.text(); - - // Parse and re-stringify JSON to sanitize whitespace - const parsedJson = JSON.parse(content); - const sanitizedJson = JSON.stringify(parsedJson); - - // For BigQuery, try to extract project_id from the credentials JSON - if (connector.name === "bigquery" && parsedJson.project_id) { - // Update the project_id field in the form - paramsForm.update( - ($form) => { - $form.project_id = parsedJson.project_id; - return $form; - }, - { taint: false }, - ); - } - - return sanitizedJson; - } catch (error) { - if (error instanceof SyntaxError) { - throw new Error(`Invalid JSON file: ${error.message}`); - } - throw new Error(`Failed to read file: ${error.message}`); - } + return formManager.handleFileUpload(file); } // Handle skip button for multi-step connectors diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 09ef5a14cb0..8004c7a121c 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -3,7 +3,22 @@ import { yup } from "sveltekit-superforms/adapters"; import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; import type { AddDataFormType } from "./types"; import { getValidationSchemaForConnector, dsnSchema } from "./FormValidation"; -import { getInitialFormValuesFromProperties } from "../sourceUtils"; +import { + getInitialFormValuesFromProperties, + inferSourceName, +} from "../sourceUtils"; +import { + submitAddConnectorForm, + submitAddSourceForm, +} from "./submitAddDataForm"; +import { normalizeConnectorError } from "./utils"; +import { MULTI_STEP_CONNECTORS } from "./constants"; +import { + connectorStepStore, + setConnectorConfig, + setStep, +} from "./connectorStepStore"; +import { get } from "svelte/store"; export class AddDataFormManager { formHeight: string; @@ -19,6 +34,8 @@ export class AddDataFormManager { // superforms instances params: ReturnType; dsn: ReturnType; + private connector: V1ConnectorDriver; + private formType: AddDataFormType; constructor(args: { connector: V1ConnectorDriver; @@ -27,6 +44,8 @@ export class AddDataFormManager { onDsnUpdate: any; }) { const { connector, formType, onParamsUpdate, onDsnUpdate } = args; + this.connector = connector; + this.formType = formType; // Layout height this.formHeight = ["clickhouse", "snowflake", "salesforce"].includes( @@ -100,4 +119,106 @@ export class AddDataFormManager { } destroy() {} + + // Business logic methods (minimal extraction) + + makeOnUpdate(args: { + onClose: () => void; + queryClient: any; + getConnectionTab: () => "parameters" | "dsn"; + setParamsError: (message: string | null, details?: string) => void; + setDsnError: (message: string | null, details?: string) => void; + }) { + const { + onClose, + queryClient, + getConnectionTab, + setParamsError, + setDsnError, + } = args; + const connector = this.connector; + const isMultiStepConnector = MULTI_STEP_CONNECTORS.includes( + connector.name ?? "", + ); + const isConnectorForm = this.formType === "connector"; + + return async (event: any) => { + if (!event.form.valid) return; + + const values = event.form.data as Record; + + try { + const stepState = get(connectorStepStore) as any; + if (isMultiStepConnector && stepState.step === "source") { + await submitAddSourceForm(queryClient, connector, values); + onClose(); + } else if (isMultiStepConnector && stepState.step === "connector") { + await submitAddConnectorForm(queryClient, connector, values, true); + setConnectorConfig(values); + setStep("source"); + return; + } else if (this.formType === "source") { + await submitAddSourceForm(queryClient, connector, values); + onClose(); + } else { + await submitAddConnectorForm(queryClient, connector, values, true); + onClose(); + } + } catch (e) { + const { message, details } = normalizeConnectorError( + connector.name ?? "", + e, + ); + const connectionTab = getConnectionTab(); + if (isConnectorForm && (this.hasOnlyDsn || connectionTab === "dsn")) { + setDsnError(message, details); + } else { + setParamsError(message, details); + } + } finally { + // no-op: saveAnyway handled in Svelte + } + }; + } + + onStringInputChange = (event: Event) => { + const target = event.target as HTMLInputElement; + const { name, value } = target; + if (name === "path") { + const tainted: any = get(this.params.tainted) as any; + if (tainted?.name) return; + const inferred = inferSourceName(this.connector, value); + if (inferred) + (this.params.form as any).update( + ($form: any) => { + $form.name = inferred; + return $form; + }, + { taint: false } as any, + ); + } + }; + + async handleFileUpload(file: File): Promise { + const content = await file.text(); + try { + const parsed = JSON.parse(content); + const sanitized = JSON.stringify(parsed); + if (this.connector.name === "bigquery" && parsed.project_id) { + (this.params.form as any).update( + ($form: any) => { + $form.project_id = parsed.project_id; + return $form; + }, + { taint: false } as any, + ); + } + return sanitized; + } catch (error: any) { + if (error instanceof SyntaxError) { + throw new Error(`Invalid JSON file: ${error.message}`); + } + throw new Error(`Failed to read file: ${error.message}`); + } + } } From 4a497b93b0d916a4e948d299915c4155922f13d4 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 31 Oct 2025 23:55:23 +0800 Subject: [PATCH 29/98] getters for source or connector form --- web-common/src/features/sources/modal/AddDataForm.svelte | 7 +++++-- .../src/features/sources/modal/AddDataFormManager.ts | 8 +++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index ad882b1825c..fac65c7921e 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -50,8 +50,8 @@ let saveAnyway = false; let showSaveAnyway = false; - const isSourceForm = formType === "source"; - const isConnectorForm = formType === "connector"; + let isSourceForm: boolean; + let isConnectorForm: boolean; let connectionTab: ConnectorType = "parameters"; @@ -75,6 +75,9 @@ onDsnUpdate: (e: any) => handleOnUpdate(e), }); + $: isSourceForm = formManager.isSourceForm; + $: isConnectorForm = formManager.isConnectorForm; + // Simple multi-step state management const isMultiStepConnector = MULTI_STEP_CONNECTORS.includes( connector.name ?? "", diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 8004c7a121c..9d99e466b29 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -120,7 +120,13 @@ export class AddDataFormManager { destroy() {} - // Business logic methods (minimal extraction) + get isSourceForm(): boolean { + return this.formType === "source"; + } + + get isConnectorForm(): boolean { + return this.formType === "connector"; + } makeOnUpdate(args: { onClose: () => void; From 0e6dbb741c008fcc81bc2dcec6cdc2576b1941bd Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Sat, 1 Nov 2025 00:08:16 +0800 Subject: [PATCH 30/98] clean up --- web-common/src/features/sources/modal/AddDataForm.svelte | 5 ----- 1 file changed, 5 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index fac65c7921e..33bc8b99eac 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -78,7 +78,6 @@ $: isSourceForm = formManager.isSourceForm; $: isConnectorForm = formManager.isConnectorForm; - // Simple multi-step state management const isMultiStepConnector = MULTI_STEP_CONNECTORS.includes( connector.name ?? "", ); @@ -106,10 +105,6 @@ paramsForm.update(() => combinedValues, { taint: false }); } - // Determine effective form type - $: effectiveFormType = - isMultiStepConnector && stepState.step === "source" ? "source" : formType; - $: formHeight = formManager.formHeight; // Form 1: Individual parameters From 44dd11407397f33913717a9773925f900569723d Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Sat, 1 Nov 2025 01:17:12 +0800 Subject: [PATCH 31/98] helpers --- .../features/sources/modal/AddDataForm.svelte | 76 +++++++------------ .../src/features/sources/modal/helpers.ts | 26 +++++++ 2 files changed, 52 insertions(+), 50 deletions(-) create mode 100644 web-common/src/features/sources/modal/helpers.ts diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 33bc8b99eac..69539c0eef1 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -7,16 +7,9 @@ import type { ActionResult } from "@sveltejs/kit"; import { createEventDispatcher } from "svelte"; import type { SuperValidated } from "sveltekit-superforms"; - import { - inferSourceName, - prepareSourceFormData, - compileSourceYAML, - } from "../sourceUtils"; + import { prepareSourceFormData, compileSourceYAML } from "../sourceUtils"; - import { - submitAddConnectorForm, - submitAddSourceForm, - } from "./submitAddDataForm"; + import { submitAddConnectorForm } from "./submitAddDataForm"; import type { AddDataFormType, ConnectorType } from "./types"; import AddClickHouseForm from "./AddClickHouseForm.svelte"; import NeedHelpText from "./NeedHelpText.svelte"; @@ -39,6 +32,7 @@ import YamlPreview from "./YamlPreview.svelte"; import GCSMultiStepForm from "./GCSMultiStepForm.svelte"; import { AddDataFormManager } from "./AddDataFormManager"; + import { hasOnlyDsn, applyClickHouseCloudRequirements } from "./helpers"; const dispatch = createEventDispatcher(); @@ -49,10 +43,8 @@ let saveAnyway = false; let showSaveAnyway = false; - let isSourceForm: boolean; let isConnectorForm: boolean; - let connectionTab: ConnectorType = "parameters"; // Wire manager-provided onUpdate after declaration below @@ -75,12 +67,15 @@ onDsnUpdate: (e: any) => handleOnUpdate(e), }); - $: isSourceForm = formManager.isSourceForm; - $: isConnectorForm = formManager.isConnectorForm; - const isMultiStepConnector = MULTI_STEP_CONNECTORS.includes( connector.name ?? "", ); + + $: isSourceForm = formManager.isSourceForm; + $: isConnectorForm = formManager.isConnectorForm; + + $: onlyDsn = hasOnlyDsn(connector, isConnectorForm); + $: stepState = $connectorStepStore; // Reactive properties based on current step @@ -151,30 +146,9 @@ let clickhouseShowSaveAnyway: boolean = false; let clickhouseHandleSaveAnyway: () => Promise; - // Helper function to check if connector only has DSN (no tabs) - function hasOnlyDsn() { - return ( - isConnectorForm && - connector.configProperties?.some((property) => property.key === "dsn") && - !connector.configProperties?.some((property) => property.key !== "dsn") - ); - } - - // Helper function to apply ClickHouse Cloud specific requirements - function applyClickHouseCloudRequirements(values: Record) { - if ( - connector.name === "clickhouse" && - clickhouseConnectorType === "clickhouse-cloud" - ) { - (values as any).ssl = true; - (values as any).port = "8443"; - } - return values; - } - // Compute disabled state for the submit button $: isSubmitDisabled = (() => { - if (hasOnlyDsn() || connectionTab === "dsn") { + if (onlyDsn || connectionTab === "dsn") { // DSN form: check required DSN properties for (const property of dsnProperties) { const key = String(property.key); @@ -211,7 +185,7 @@ })(); $: formId = (() => { - if (hasOnlyDsn() || connectionTab === "dsn") { + if (onlyDsn || connectionTab === "dsn") { return dsnFormId; } else { return paramsFormId; @@ -219,7 +193,7 @@ })(); $: submitting = (() => { - if (hasOnlyDsn() || connectionTab === "dsn") { + if (onlyDsn || connectionTab === "dsn") { return $dsnSubmitting; } else { return $paramsSubmitting; @@ -228,7 +202,7 @@ // Reset errors when form is modified $: (() => { - if (hasOnlyDsn() || connectionTab === "dsn") { + if (onlyDsn || connectionTab === "dsn") { if ($dsnTainted) dsnError = null; } else { if ($paramsTainted) paramsError = null; @@ -267,11 +241,14 @@ saveAnyway = true; // Get the current form values based on the active form - const values = - hasOnlyDsn() || connectionTab === "dsn" ? $dsnForm : $paramsForm; + const values = onlyDsn || connectionTab === "dsn" ? $dsnForm : $paramsForm; // Apply ClickHouse Cloud requirements if needed - const processedValues = applyClickHouseCloudRequirements(values); + const processedValues = applyClickHouseCloudRequirements( + connector.name, + clickhouseConnectorType, + values, + ); try { // Only call submitAddConnectorForm since Save Anyway is connector-only @@ -289,7 +266,7 @@ ); // Keep error state for each form - match the display logic - if (hasOnlyDsn() || connectionTab === "dsn") { + if (onlyDsn || connectionTab === "dsn") { dsnError = message; dsnErrorDetails = details; } else { @@ -320,7 +297,7 @@ fieldFilter: (property) => { // When in DSN mode, don't filter out noPrompt properties // because the DSN field itself might have noPrompt: true - if (hasOnlyDsn() || connectionTab === "dsn") { + if (onlyDsn || connectionTab === "dsn") { return true; // Show all DSN properties } return !property.noPrompt; @@ -337,13 +314,13 @@ fieldFilter: (property) => { // When in DSN mode, don't filter out noPrompt properties // because the DSN field itself might have noPrompt: true - if (hasOnlyDsn() || connectionTab === "dsn") { + if (onlyDsn || connectionTab === "dsn") { return true; // Show all DSN properties } return !property.noPrompt; }, orderedProperties: - hasOnlyDsn() || connectionTab === "dsn" + onlyDsn || connectionTab === "dsn" ? filteredDsnProperties : filteredParamsProperties, }); @@ -403,8 +380,7 @@ } } - const values = - hasOnlyDsn() || connectionTab === "dsn" ? $dsnForm : $paramsForm; + const values = onlyDsn || connectionTab === "dsn" ? $dsnForm : $paramsForm; if (isConnectorForm) { // Connector form @@ -681,10 +657,10 @@ {#if dsnError || paramsError || clickhouseError} p.key === "dsn"); + const hasOthers = props.some((p) => p.key !== "dsn"); + return hasDsn && !hasOthers; +} + +export function applyClickHouseCloudRequirements( + connectorName: string | undefined, + connectorType: ClickHouseConnectorType, + values: Record, +): Record { + if (connectorName === "clickhouse" && connectorType === "clickhouse-cloud") { + return { ...values, ssl: true, port: "8443" } as Record; + } + return values; +} + + From b6fd63c01cde4167dc72c32bd8100a9d6ab7b397 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Sat, 1 Nov 2025 01:52:21 +0800 Subject: [PATCH 32/98] extract yaml preview --- .../features/sources/modal/AddDataForm.svelte | 175 +++--------------- .../sources/modal/AddDataFormManager.ts | 160 ++++++++++++++++ 2 files changed, 188 insertions(+), 147 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 69539c0eef1..00ef53293f6 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -7,7 +7,6 @@ import type { ActionResult } from "@sveltejs/kit"; import { createEventDispatcher } from "svelte"; import type { SuperValidated } from "sveltekit-superforms"; - import { prepareSourceFormData, compileSourceYAML } from "../sourceUtils"; import { submitAddConnectorForm } from "./submitAddDataForm"; import type { AddDataFormType, ConnectorType } from "./types"; @@ -21,7 +20,7 @@ type ClickHouseConnectorType, } from "./constants"; import { getInitialFormValuesFromProperties } from "../sourceUtils"; - import { compileConnectorYAML } from "../../connectors/code-utils"; + import { MULTI_STEP_CONNECTORS } from "./constants"; import { connectorStepStore, @@ -32,7 +31,7 @@ import YamlPreview from "./YamlPreview.svelte"; import GCSMultiStepForm from "./GCSMultiStepForm.svelte"; import { AddDataFormManager } from "./AddDataFormManager"; - import { hasOnlyDsn, applyClickHouseCloudRequirements } from "./helpers"; + import { hasOnlyDsn } from "./helpers"; const dispatch = createEventDispatcher(); @@ -222,7 +221,6 @@ } })(); - // Emit the submitting state to the parent $: dispatch("submitting", { submitting }); async function handleSaveAnyway() { @@ -237,159 +235,42 @@ return; } - // For other connectors, use the original logic + // For other connectors, use manager helper saveAnyway = true; - - // Get the current form values based on the active form const values = onlyDsn || connectionTab === "dsn" ? $dsnForm : $paramsForm; - - // Apply ClickHouse Cloud requirements if needed - const processedValues = applyClickHouseCloudRequirements( - connector.name, - clickhouseConnectorType, + const result = await formManager.saveConnectorAnyway({ + queryClient, values, - ); - - try { - // Only call submitAddConnectorForm since Save Anyway is connector-only - await submitAddConnectorForm( - queryClient, - connector, - processedValues, - true, - ); + clickhouseConnectorType, + }); + if (result.ok) { onClose(); - } catch (e) { - const { message, details } = normalizeConnectorError( - connector.name ?? "", - e, - ); - - // Keep error state for each form - match the display logic + } else { if (onlyDsn || connectionTab === "dsn") { - dsnError = message; - dsnErrorDetails = details; + dsnError = result.message; + dsnErrorDetails = result.details; } else { - paramsError = message; - paramsErrorDetails = details; + paramsError = result.message; + paramsErrorDetails = result.details; } - } finally { - // Reset saveAnyway state after submission completes - saveAnyway = false; } + saveAnyway = false; } - function getClickHouseYamlPreview( - values: Record, - connectorType: ClickHouseConnectorType, - ) { - // Convert connectorType to managed boolean for YAML compatibility - const managed = connectorType === "rill-managed"; - - // Ensure ClickHouse Cloud specific requirements are met in preview - const previewValues = { ...values, managed } as Record; - if (connectorType === "clickhouse-cloud") { - previewValues.ssl = true; - previewValues.port = "8443"; - } - - return compileConnectorYAML(connector, previewValues, { - fieldFilter: (property) => { - // When in DSN mode, don't filter out noPrompt properties - // because the DSN field itself might have noPrompt: true - if (onlyDsn || connectionTab === "dsn") { - return true; // Show all DSN properties - } - return !property.noPrompt; - }, - orderedProperties: - connectionTab === "dsn" - ? filteredDsnProperties - : filteredParamsProperties, - }); - } - - function getConnectorYamlPreview(values: Record) { - return compileConnectorYAML(connector, values, { - fieldFilter: (property) => { - // When in DSN mode, don't filter out noPrompt properties - // because the DSN field itself might have noPrompt: true - if (onlyDsn || connectionTab === "dsn") { - return true; // Show all DSN properties - } - return !property.noPrompt; - }, - orderedProperties: - onlyDsn || connectionTab === "dsn" - ? filteredDsnProperties - : filteredParamsProperties, - }); - } - - function getSourceYamlPreview(values: Record) { - // For multi-step connectors in step 2, filter out connector properties - let filteredValues = values; - if (isMultiStepConnector && stepState.step === "source") { - // Get connector property keys to filter out - const connectorPropertyKeys = new Set( - connector.configProperties?.map((prop) => prop.key).filter(Boolean) || - [], - ); - - // Filter out connector properties, keeping only source properties and other necessary fields - filteredValues = Object.fromEntries( - Object.entries(values).filter( - ([key]) => !connectorPropertyKeys.has(key), - ), - ); - } - - const [rewrittenConnector, rewrittenFormValues] = prepareSourceFormData( - connector, - filteredValues, - ); - - // Check if the connector was rewritten to DuckDB - const isRewrittenToDuckDb = rewrittenConnector.name === "duckdb"; - - if (isRewrittenToDuckDb) { - return compileSourceYAML(rewrittenConnector, rewrittenFormValues); - } else { - return getConnectorYamlPreview(rewrittenFormValues); - } - } - - $: yamlPreview = (() => { - // ClickHouse special case - if (connector.name === "clickhouse") { - // Reactive form values - const values = - connectionTab === "dsn" ? $clickhouseDsnForm : $clickhouseParamsForm; - return getClickHouseYamlPreview(values, clickhouseConnectorType); - } - - // Multi-step connector special case - show different preview based on step - if (isMultiStepConnector) { - if (stepState.step === "connector") { - // Step 1: Show connector preview - return getConnectorYamlPreview($paramsForm); - } else { - // Step 2: Show source preview with stored connector config - const combinedValues = { ...stepState.connectorConfig, ...$paramsForm }; - return getSourceYamlPreview(combinedValues); - } - } - - const values = onlyDsn || connectionTab === "dsn" ? $dsnForm : $paramsForm; - - if (isConnectorForm) { - // Connector form - return getConnectorYamlPreview(values); - } else { - // Source form - return getSourceYamlPreview(values); - } - })(); + $: yamlPreview = formManager.computeYamlPreview({ + connectionTab, + onlyDsn, + filteredParamsProperties, + filteredDsnProperties, + stepState, + isMultiStepConnector, + isConnectorForm, + paramsFormValues: $paramsForm, + dsnFormValues: $dsnForm, + clickhouseConnectorType, + clickhouseParamsValues: $clickhouseParamsForm, + clickhouseDsnValues: $clickhouseDsnForm, + }); $: isClickhouse = connector.name === "clickhouse"; diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 9d99e466b29..d0c5417f024 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -19,6 +19,11 @@ import { setStep, } from "./connectorStepStore"; import { get } from "svelte/store"; +import { compileConnectorYAML } from "../../connectors/code-utils"; +import { compileSourceYAML, prepareSourceFormData } from "../sourceUtils"; +import type { ConnectorDriverProperty } from "@rilldata/web-common/runtime-client"; +import type { ClickHouseConnectorType } from "./constants"; +import { applyClickHouseCloudRequirements } from "./helpers"; export class AddDataFormManager { formHeight: string; @@ -227,4 +232,159 @@ export class AddDataFormManager { throw new Error(`Failed to read file: ${error.message}`); } } + + /** + * Compute YAML preview for the current form state. + */ + computeYamlPreview(ctx: { + connectionTab: "parameters" | "dsn"; + onlyDsn: boolean; + filteredParamsProperties: ConnectorDriverProperty[]; + filteredDsnProperties: ConnectorDriverProperty[]; + stepState: any; + isMultiStepConnector: boolean; + isConnectorForm: boolean; + paramsFormValues: Record; + dsnFormValues: Record; + clickhouseConnectorType?: ClickHouseConnectorType; + clickhouseParamsValues?: Record; + clickhouseDsnValues?: Record; + }): string { + const connector = this.connector; + const { + connectionTab, + onlyDsn, + filteredParamsProperties, + filteredDsnProperties, + stepState, + isMultiStepConnector, + isConnectorForm, + paramsFormValues, + dsnFormValues, + clickhouseConnectorType, + clickhouseParamsValues, + clickhouseDsnValues, + } = ctx; + + const getConnectorYamlPreview = (values: Record) => { + return compileConnectorYAML(connector, values, { + fieldFilter: (property) => { + if (onlyDsn || connectionTab === "dsn") return true; + return !property.noPrompt; + }, + orderedProperties: + onlyDsn || connectionTab === "dsn" + ? filteredDsnProperties + : filteredParamsProperties, + }); + }; + + const getClickHouseYamlPreview = ( + values: Record, + chType: ClickHouseConnectorType | undefined, + ) => { + // Convert to managed boolean and apply CH Cloud requirements for preview + const managed = chType === "rill-managed"; + const previewValues = { ...values, managed } as Record; + const finalValues = applyClickHouseCloudRequirements( + connector.name, + chType as ClickHouseConnectorType, + previewValues, + ); + return compileConnectorYAML(connector, finalValues, { + fieldFilter: (property) => { + if (onlyDsn || connectionTab === "dsn") return true; + return !property.noPrompt; + }, + orderedProperties: + connectionTab === "dsn" + ? filteredDsnProperties + : filteredParamsProperties, + }); + }; + + const getSourceYamlPreview = (values: Record) => { + // For multi-step connectors in step 2, filter out connector properties + let filteredValues = values; + if (isMultiStepConnector && stepState?.step === "source") { + const connectorPropertyKeys = new Set( + connector.configProperties?.map((p) => p.key).filter(Boolean) || [], + ); + filteredValues = Object.fromEntries( + Object.entries(values).filter( + ([key]) => !connectorPropertyKeys.has(key), + ), + ); + } + + const [rewrittenConnector, rewrittenFormValues] = prepareSourceFormData( + connector, + filteredValues, + ); + const isRewrittenToDuckDb = rewrittenConnector.name === "duckdb"; + if (isRewrittenToDuckDb) { + return compileSourceYAML(rewrittenConnector, rewrittenFormValues); + } + return getConnectorYamlPreview(rewrittenFormValues); + }; + + // ClickHouse special-case + if (connector.name === "clickhouse") { + const values = + connectionTab === "dsn" + ? clickhouseDsnValues || {} + : clickhouseParamsValues || {}; + return getClickHouseYamlPreview(values, clickhouseConnectorType); + } + + // Multi-step connectors + if (isMultiStepConnector) { + if (stepState?.step === "connector") { + return getConnectorYamlPreview(paramsFormValues); + } else { + const combinedValues = { + ...(stepState?.connectorConfig || {}), + ...paramsFormValues, + } as Record; + return getSourceYamlPreview(combinedValues); + } + } + + const currentValues = + onlyDsn || connectionTab === "dsn" ? dsnFormValues : paramsFormValues; + if (isConnectorForm) return getConnectorYamlPreview(currentValues); + return getSourceYamlPreview(currentValues); + } + + /** + * Save connector anyway (non-ClickHouse), returning a result object for the caller to handle. + */ + async saveConnectorAnyway(args: { + queryClient: any; + values: Record; + clickhouseConnectorType?: ClickHouseConnectorType; + }): Promise<{ ok: true } | { ok: false; message: string; details?: string }> { + const { queryClient, values, clickhouseConnectorType } = args; + const processedValues = applyClickHouseCloudRequirements( + this.connector.name, + (clickhouseConnectorType as ClickHouseConnectorType) || + ("self-hosted" as ClickHouseConnectorType), + values, + ); + try { + await submitAddConnectorForm( + queryClient, + this.connector, + processedValues, + true, + ); + return { ok: true } as const; + } catch (e) { + const { message, details } = normalizeConnectorError( + this.connector.name ?? "", + e, + ); + return { ok: false, message, details } as const; + } + } } From 38b2c420f78107ca2d1fb2b329d1953a81c0d351 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Sat, 1 Nov 2025 01:57:20 +0800 Subject: [PATCH 33/98] multi step detection --- .../features/sources/modal/AddDataForm.svelte | 17 +++-------------- .../sources/modal/AddDataFormManager.ts | 4 ++++ 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 00ef53293f6..461c04780f0 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -21,7 +21,6 @@ } from "./constants"; import { getInitialFormValuesFromProperties } from "../sourceUtils"; - import { MULTI_STEP_CONNECTORS } from "./constants"; import { connectorStepStore, setStep, @@ -58,7 +57,6 @@ result: Extract; }) => Promise; - // Form Generation Manager (phase 1) const formManager = new AddDataFormManager({ connector, formType, @@ -66,18 +64,14 @@ onDsnUpdate: (e: any) => handleOnUpdate(e), }); - const isMultiStepConnector = MULTI_STEP_CONNECTORS.includes( - connector.name ?? "", - ); + $: isMultiStepConnector = formManager.isMultiStepConnector; + + $: dispatch("submitting", { submitting }); $: isSourceForm = formManager.isSourceForm; $: isConnectorForm = formManager.isConnectorForm; - $: onlyDsn = hasOnlyDsn(connector, isConnectorForm); - $: stepState = $connectorStepStore; - - // Reactive properties based on current step $: stepProperties = isMultiStepConnector && stepState.step === "source" ? (connector.sourceProperties ?? []) @@ -98,7 +92,6 @@ paramsForm.update(() => combinedValues, { taint: false }); } - $: formHeight = formManager.formHeight; // Form 1: Individual parameters @@ -135,7 +128,6 @@ let clickhouseError: string | null = null; let clickhouseErrorDetails: string | undefined = undefined; - let clickhouseFormId: string = ""; let clickhouseSubmitting: boolean; let clickhouseIsSubmitDisabled: boolean; @@ -145,7 +137,6 @@ let clickhouseShowSaveAnyway: boolean = false; let clickhouseHandleSaveAnyway: () => Promise; - // Compute disabled state for the submit button $: isSubmitDisabled = (() => { if (onlyDsn || connectionTab === "dsn") { // DSN form: check required DSN properties @@ -221,8 +212,6 @@ } })(); - $: dispatch("submitting", { submitting }); - async function handleSaveAnyway() { // Save Anyway should only work for connector forms if (!isConnectorForm) { diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index d0c5417f024..7d72a433b4a 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -133,6 +133,10 @@ export class AddDataFormManager { return this.formType === "connector"; } + get isMultiStepConnector(): boolean { + return MULTI_STEP_CONNECTORS.includes(this.connector.name ?? ""); + } + makeOnUpdate(args: { onClose: () => void; queryClient: any; From 4a16f0b83bc757416db7ff3be832d6ba8732023b Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Sat, 1 Nov 2025 02:01:38 +0800 Subject: [PATCH 34/98] handle back and skip to manager --- .../features/sources/modal/AddDataForm.svelte | 38 +++++-------------- .../sources/modal/AddDataFormManager.ts | 16 ++++++++ 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 461c04780f0..35449f158c0 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -21,11 +21,7 @@ } from "./constants"; import { getInitialFormValuesFromProperties } from "../sourceUtils"; - import { - connectorStepStore, - setStep, - setConnectorConfig, - } from "./connectorStepStore"; + import { connectorStepStore } from "./connectorStepStore"; import FormRenderer from "./FormRenderer.svelte"; import YamlPreview from "./YamlPreview.svelte"; import GCSMultiStepForm from "./GCSMultiStepForm.svelte"; @@ -270,10 +266,6 @@ ? clickhouseSubmitting && saveAnyway : submitting && saveAnyway; - function onStringInputChange(event: Event) { - formManager.onStringInputChange(event); - } - handleOnUpdate = formManager.makeOnUpdate({ onClose, queryClient, @@ -288,28 +280,12 @@ }, }); - // Handle file upload for credential files async function handleFileUpload(file: File): Promise { return formManager.handleFileUpload(file); } - // Handle skip button for multi-step connectors - function handleSkip() { - if (!isMultiStepConnector || stepState.step !== "connector") return; - - // Store current form values and transition to step 2 - setConnectorConfig($paramsForm); - setStep("source"); - } - - function handleBack() { - if (isMultiStepConnector && stepState.step === "source") { - // Go back to step 1 (connector configuration) - setStep("connector"); - } else { - // Use the original back behavior for non-multi-step or step 1 - onBack(); - } + function onStringInputChange(event: Event) { + formManager.onStringInputChange(event); } @@ -449,7 +425,9 @@
- +
{#if shouldShowSaveAnywayButton} @@ -465,7 +443,9 @@ {/if} {#if isMultiStepConnector && stepState.step === "connector"} - + {/if}
diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 828773af765..8beffee3504 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -163,6 +163,36 @@ export class AddDataFormManager { } } + getPrimaryButtonLabel(args: { + isConnectorForm: boolean; + step: "connector" | "source" | string; + submitting: boolean; + clickhouseConnectorType?: ClickHouseConnectorType; + clickhouseSubmitting?: boolean; + }): string { + const { isConnectorForm, step, submitting, clickhouseConnectorType, clickhouseSubmitting } = args; + const isClickhouse = this.connector.name === "clickhouse"; + + if (isClickhouse) { + if (clickhouseConnectorType === "rill-managed") { + return clickhouseSubmitting ? "Connecting..." : "Connect"; + } + return clickhouseSubmitting ? "Testing connection..." : "Test and Connect"; + } + + if (isConnectorForm) { + if (this.isMultiStepConnector && step === "connector") { + return submitting ? "Testing connection..." : "Test and Connect"; + } + if (this.isMultiStepConnector && step === "source") { + return submitting ? "Creating model..." : "Test and Add data"; + } + return submitting ? "Testing connection..." : "Test and Connect"; + } + + return "Test and Add data"; + } + makeOnUpdate(args: { onClose: () => void; queryClient: any; From dd41b76fc5c3b066bb9959ecdf0c6226b59ad982 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 4 Nov 2025 23:51:40 +0800 Subject: [PATCH 37/98] prettier --- .../src/features/sources/modal/AddDataFormManager.ts | 12 ++++++++++-- web-common/src/features/sources/modal/helpers.ts | 2 -- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 8beffee3504..cd9aae23c84 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -170,14 +170,22 @@ export class AddDataFormManager { clickhouseConnectorType?: ClickHouseConnectorType; clickhouseSubmitting?: boolean; }): string { - const { isConnectorForm, step, submitting, clickhouseConnectorType, clickhouseSubmitting } = args; + const { + isConnectorForm, + step, + submitting, + clickhouseConnectorType, + clickhouseSubmitting, + } = args; const isClickhouse = this.connector.name === "clickhouse"; if (isClickhouse) { if (clickhouseConnectorType === "rill-managed") { return clickhouseSubmitting ? "Connecting..." : "Connect"; } - return clickhouseSubmitting ? "Testing connection..." : "Test and Connect"; + return clickhouseSubmitting + ? "Testing connection..." + : "Test and Connect"; } if (isConnectorForm) { diff --git a/web-common/src/features/sources/modal/helpers.ts b/web-common/src/features/sources/modal/helpers.ts index a0fb123e3e6..0829a2f5067 100644 --- a/web-common/src/features/sources/modal/helpers.ts +++ b/web-common/src/features/sources/modal/helpers.ts @@ -22,5 +22,3 @@ export function applyClickHouseCloudRequirements( } return values; } - - From 6dec4d4a1db05ec005613ad2e74589d504db3ae4 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 5 Nov 2025 00:01:04 +0800 Subject: [PATCH 38/98] lint --- web-common/src/features/sources/modal/AddDataForm.svelte | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index c1484af3d3d..c91626d1f04 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -8,13 +8,12 @@ import { createEventDispatcher } from "svelte"; import type { SuperValidated } from "sveltekit-superforms"; - import { submitAddConnectorForm } from "./submitAddDataForm"; import type { AddDataFormType, ConnectorType } from "./types"; import AddClickHouseForm from "./AddClickHouseForm.svelte"; import NeedHelpText from "./NeedHelpText.svelte"; import Tabs from "@rilldata/web-common/components/forms/Tabs.svelte"; import { TabsContent } from "@rilldata/web-common/components/tabs"; - import { isEmpty, normalizeConnectorError } from "./utils"; + import { isEmpty } from "./utils"; import { CONNECTION_TAB_OPTIONS, type ClickHouseConnectorType, From 6ba81ca297632fc4a3697586c736c6c7ebd4c038 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 5 Nov 2025 00:55:28 +0800 Subject: [PATCH 39/98] fix conflicts --- .../features/sources/modal/AddClickHouseForm.svelte | 6 ++---- .../src/features/sources/modal/AddDataForm.svelte | 11 ++++------- .../src/features/sources/modal/AddDataModal.svelte | 6 +----- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/web-common/src/features/sources/modal/AddClickHouseForm.svelte b/web-common/src/features/sources/modal/AddClickHouseForm.svelte index f90be8ee018..6fff58af56a 100644 --- a/web-common/src/features/sources/modal/AddClickHouseForm.svelte +++ b/web-common/src/features/sources/modal/AddClickHouseForm.svelte @@ -32,7 +32,7 @@ export let connector: V1ConnectorDriver; export let formId: string; - export let submitting: boolean; + export let isSubmitting: boolean; export let isSubmitDisabled: boolean; export let connectorType: ClickHouseConnectorType = "self-hosted"; export let connectionTab: ConnectorType = "parameters"; @@ -93,6 +93,7 @@ $: submitting = connectionTab === "parameters" ? $paramsSubmitting : $dsnSubmitting; + $: isSubmitting = submitting; $: formId = connectionTab === "parameters" ? paramsFormId : dsnFormId; // Reset connectionTab if switching to Rill-managed @@ -116,9 +117,6 @@ dsnErrorDetails = undefined; } - // Emit the submitting state to the parent - $: dispatch("submitting", { submitting }); - let prevConnectorType = connectorType; $: { // Switching to Rill-managed: set managed=true and clear other properties diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index c91626d1f04..162e5121573 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -5,7 +5,6 @@ import { queryClient } from "@rilldata/web-common/lib/svelte-query/globalQueryClient"; import { type V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; import type { ActionResult } from "@sveltejs/kit"; - import { createEventDispatcher } from "svelte"; import type { SuperValidated } from "sveltekit-superforms"; import type { AddDataFormType, ConnectorType } from "./types"; @@ -27,10 +26,9 @@ import { AddDataFormManager } from "./AddDataFormManager"; import { hasOnlyDsn } from "./helpers"; - const dispatch = createEventDispatcher(); - export let connector: V1ConnectorDriver; export let formType: AddDataFormType; + export let isSubmitting: boolean; export let onBack: () => void; export let onClose: () => void; @@ -61,8 +59,6 @@ $: isMultiStepConnector = formManager.isMultiStepConnector; - $: dispatch("submitting", { submitting }); - $: isSourceForm = formManager.isSourceForm; $: isConnectorForm = formManager.isConnectorForm; $: onlyDsn = hasOnlyDsn(connector, isConnectorForm); @@ -179,6 +175,8 @@ } })(); + $: isSubmitting = submitting; + // Reset errors when form is modified $: (() => { if (onlyDsn || connectionTab === "dsn") { @@ -294,7 +292,7 @@ clickhouseErrorDetails = details; }} bind:formId={clickhouseFormId} - bind:submitting={clickhouseSubmitting} + bind:isSubmitting={clickhouseSubmitting} bind:isSubmitDisabled={clickhouseIsSubmitDisabled} bind:connectorType={clickhouseConnectorType} bind:connectionTab @@ -302,7 +300,6 @@ bind:dsnForm={clickhouseDsnForm} bind:showSaveAnyway={clickhouseShowSaveAnyway} bind:handleSaveAnyway={clickhouseHandleSaveAnyway} - on:submitting /> {:else if hasDsnFormOption} {/if} {/if} From 2f64226549a380c2c688949b71cbee168fb83d4d Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 5 Nov 2025 01:06:07 +0800 Subject: [PATCH 40/98] hoist form section component, reduce loc --- .../features/sources/modal/AddDataForm.svelte | 55 ++++++++----------- .../features/sources/modal/FormSection.svelte | 14 +++++ 2 files changed, 37 insertions(+), 32 deletions(-) create mode 100644 web-common/src/features/sources/modal/FormSection.svelte diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 162e5121573..39b2caa2ecb 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -25,6 +25,7 @@ import GCSMultiStepForm from "./GCSMultiStepForm.svelte"; import { AddDataFormManager } from "./AddDataFormManager"; import { hasOnlyDsn } from "./helpers"; + import FormSection from "./FormSection.svelte"; export let connector: V1ConnectorDriver; export let formType: AddDataFormType; @@ -308,11 +309,10 @@ disableMarginTop > -
- +
-
- +
{:else if isConnectorForm && connector.configProperties?.some((property) => property.key === "dsn")} -
+ - +
{:else if isMultiStepConnector} {#if stepState.step === "connector"} -
- + {:else} -
- + {/if} {:else} -
- + {/if}
diff --git a/web-common/src/features/sources/modal/FormSection.svelte b/web-common/src/features/sources/modal/FormSection.svelte new file mode 100644 index 00000000000..f35ed109ead --- /dev/null +++ b/web-common/src/features/sources/modal/FormSection.svelte @@ -0,0 +1,14 @@ + + +
+ + From d140bf24ff0b03808d73193371f7461f5adeb6e3 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 11 Nov 2025 14:11:48 +0800 Subject: [PATCH 41/98] clean up --- web-common/src/features/sources/modal/AddDataForm.svelte | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 39b2caa2ecb..2180a1496a7 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -59,7 +59,6 @@ }); $: isMultiStepConnector = formManager.isMultiStepConnector; - $: isSourceForm = formManager.isSourceForm; $: isConnectorForm = formManager.isConnectorForm; $: onlyDsn = hasOnlyDsn(connector, isConnectorForm); @@ -68,8 +67,7 @@ isMultiStepConnector && stepState.step === "source" ? (connector.sourceProperties ?? []) : properties; - - // Update form when transitioning to step 2 + $: formHeight = formManager.formHeight; $: if ( isMultiStepConnector && stepState.step === "source" && @@ -84,7 +82,6 @@ paramsForm.update(() => combinedValues, { taint: false }); } - $: formHeight = formManager.formHeight; // Form 1: Individual parameters const paramsFormId = formManager.paramsFormId; From 1b5c2adafe050d40e1e02faeb027b628ad6e5a13 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 11 Nov 2025 14:28:53 +0800 Subject: [PATCH 42/98] use property type --- .../src/features/sources/modal/AddDataFormManager.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index cd9aae23c84..3cb2a49e18e 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -31,10 +31,10 @@ export class AddDataFormManager { dsnFormId: string; hasDsnFormOption: boolean; hasOnlyDsn: boolean; - properties: any[]; - filteredParamsProperties: any[]; - dsnProperties: any[]; - filteredDsnProperties: any[]; + properties: ConnectorDriverProperty[]; + filteredParamsProperties: ConnectorDriverProperty[]; + dsnProperties: ConnectorDriverProperty[]; + filteredDsnProperties: ConnectorDriverProperty[]; // superforms instances params: ReturnType; From 2044fa87ef666ce2057eea8310c3d1402b218119 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 11 Nov 2025 14:39:50 +0800 Subject: [PATCH 43/98] redundant reactive statement --- web-common/src/features/sources/modal/AddDataForm.svelte | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 2180a1496a7..b52a8716001 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -67,7 +67,6 @@ isMultiStepConnector && stepState.step === "source" ? (connector.sourceProperties ?? []) : properties; - $: formHeight = formManager.formHeight; $: if ( isMultiStepConnector && stepState.step === "source" && @@ -280,7 +279,9 @@
-
+
{#if connector.name === "clickhouse"} Date: Tue, 11 Nov 2025 15:09:12 +0800 Subject: [PATCH 44/98] consolidate save anyway code path --- .../features/sources/modal/AddDataForm.svelte | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index b52a8716001..eb0bae0186a 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -123,7 +123,6 @@ let clickhouseParamsForm; let clickhouseDsnForm; let clickhouseShowSaveAnyway: boolean = false; - let clickhouseHandleSaveAnyway: () => Promise; $: isSubmitDisabled = (() => { if (onlyDsn || connectionTab === "dsn") { @@ -202,15 +201,19 @@ return; } - // For ClickHouse, delegate to the child component's handleSaveAnyway function - if (connector.name === "clickhouse") { - await clickhouseHandleSaveAnyway(); - return; - } - // For other connectors, use manager helper saveAnyway = true; - const values = onlyDsn || connectionTab === "dsn" ? $dsnForm : $paramsForm; + const values = + connector.name === "clickhouse" + ? connectionTab === "dsn" + ? $clickhouseDsnForm + : $clickhouseParamsForm + : onlyDsn || connectionTab === "dsn" + ? $dsnForm + : $paramsForm; + if (connector.name === "clickhouse") { + clickhouseSubmitting = true; + } const result = await formManager.saveConnectorAnyway({ queryClient, values, @@ -219,7 +222,15 @@ if (result.ok) { onClose(); } else { - if (onlyDsn || connectionTab === "dsn") { + if (connector.name === "clickhouse") { + if (connectionTab === "dsn") { + dsnError = result.message; + dsnErrorDetails = result.details; + } else { + paramsError = result.message; + paramsErrorDetails = result.details; + } + } else if (onlyDsn || connectionTab === "dsn") { dsnError = result.message; dsnErrorDetails = result.details; } else { @@ -228,6 +239,9 @@ } } saveAnyway = false; + if (connector.name === "clickhouse") { + clickhouseSubmitting = false; + } } $: yamlPreview = formManager.computeYamlPreview({ @@ -298,7 +312,6 @@ bind:paramsForm={clickhouseParamsForm} bind:dsnForm={clickhouseDsnForm} bind:showSaveAnyway={clickhouseShowSaveAnyway} - bind:handleSaveAnyway={clickhouseHandleSaveAnyway} /> {:else if hasDsnFormOption} Date: Tue, 11 Nov 2025 16:26:06 +0800 Subject: [PATCH 45/98] clean up --- web-common/src/features/sources/modal/AddDataFormManager.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 3cb2a49e18e..09f99a981fd 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -123,8 +123,6 @@ export class AddDataFormManager { } as any); } - destroy() {} - get isSourceForm(): boolean { return this.formType === "source"; } From a93c0dcd355c34d6b7e7ee8fb0143d5d3342a822 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 11 Nov 2025 16:37:23 +0800 Subject: [PATCH 46/98] remove reactive statements, use const --- .../src/features/sources/modal/AddDataForm.svelte | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index eb0bae0186a..e42702c2954 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -35,8 +35,6 @@ let saveAnyway = false; let showSaveAnyway = false; - let isSourceForm: boolean; - let isConnectorForm: boolean; let connectionTab: ConnectorType = "parameters"; // Wire manager-provided onUpdate after declaration below @@ -58,10 +56,10 @@ onDsnUpdate: (e: any) => handleOnUpdate(e), }); - $: isMultiStepConnector = formManager.isMultiStepConnector; - $: isSourceForm = formManager.isSourceForm; - $: isConnectorForm = formManager.isConnectorForm; - $: onlyDsn = hasOnlyDsn(connector, isConnectorForm); + const isMultiStepConnector = formManager.isMultiStepConnector; + const isSourceForm = formManager.isSourceForm; + const isConnectorForm = formManager.isConnectorForm; + const onlyDsn = hasOnlyDsn(connector, isConnectorForm); $: stepState = $connectorStepStore; $: stepProperties = isMultiStepConnector && stepState.step === "source" From b27238fdd6227170384753778418aedf7cb4cf86 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 11 Nov 2025 16:46:16 +0800 Subject: [PATCH 47/98] fix 2 --- web-common/src/features/sources/modal/AddDataForm.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index e42702c2954..de5f4513a61 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -47,7 +47,7 @@ formEl: HTMLFormElement; cancel: () => void; result: Extract; - }) => Promise; + }) => Promise = async (_event) => {}; const formManager = new AddDataFormManager({ connector, From e9be431db680c92f11d9026c3311f18b0a7cdace Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 11 Nov 2025 16:49:37 +0800 Subject: [PATCH 48/98] fix duplicate error handling --- .../features/sources/modal/AddDataFormManager.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 09f99a981fd..2411a0635ea 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -42,6 +42,11 @@ export class AddDataFormManager { private connector: V1ConnectorDriver; private formType: AddDataFormType; + // Centralized error normalization for this manager + private normalizeError(e: unknown): { message: string; details?: string } { + return normalizeConnectorError(this.connector.name ?? "", e); + } + constructor(args: { connector: V1ConnectorDriver; formType: AddDataFormType; @@ -242,10 +247,7 @@ export class AddDataFormManager { onClose(); } } catch (e) { - const { message, details } = normalizeConnectorError( - connector.name ?? "", - e, - ); + const { message, details } = this.normalizeError(e); const connectionTab = getConnectionTab(); if (isConnectorForm && (this.hasOnlyDsn || connectionTab === "dsn")) { setDsnError(message, details); @@ -446,10 +448,7 @@ export class AddDataFormManager { ); return { ok: true } as const; } catch (e) { - const { message, details } = normalizeConnectorError( - this.connector.name ?? "", - e, - ); + const { message, details } = this.normalizeError(e); return { ok: false, message, details } as const; } } From 486b1e3339d9a0328e3368e77756850616d655ae Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 11 Nov 2025 22:36:16 +0800 Subject: [PATCH 49/98] fix tainted --- .../src/features/sources/modal/AddDataForm.svelte | 2 +- .../src/features/sources/modal/AddDataFormManager.ts | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index de5f4513a61..d0e703d767c 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -282,7 +282,7 @@ } function onStringInputChange(event: Event) { - formManager.onStringInputChange(event); + formManager.onStringInputChange(event, $paramsTainted); } diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 2411a0635ea..5dcb1cd152d 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -260,12 +260,18 @@ export class AddDataFormManager { }; } - onStringInputChange = (event: Event) => { + onStringInputChange = ( + event: Event, + taintedFields?: Record | null, + ) => { const target = event.target as HTMLInputElement; const { name, value } = target; if (name === "path") { - const tainted: any = get(this.params.tainted) as any; - if (tainted?.name) return; + const nameTainted = + taintedFields && typeof taintedFields === "object" + ? Boolean((taintedFields as any)?.name) + : false; + if (nameTainted) return; const inferred = inferSourceName(this.connector, value); if (inferred) (this.params.form as any).update( From 143a32f9c5edd7d9fff8505a6c111eaff1b30dee Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 11 Nov 2025 22:41:18 +0800 Subject: [PATCH 50/98] extract values to constants --- .../features/sources/modal/AddDataFormManager.ts | 15 +++++++++------ .../src/features/sources/modal/constants.ts | 8 ++++++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 5dcb1cd152d..536e6418e8e 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -12,7 +12,12 @@ import { submitAddSourceForm, } from "./submitAddDataForm"; import { normalizeConnectorError } from "./utils"; -import { MULTI_STEP_CONNECTORS } from "./constants"; +import { + FORM_HEIGHT_DEFAULT, + FORM_HEIGHT_TALL, + MULTI_STEP_CONNECTORS, + TALL_FORM_CONNECTORS, +} from "./constants"; import { connectorStepStore, setConnectorConfig, @@ -58,11 +63,9 @@ export class AddDataFormManager { this.formType = formType; // Layout height - this.formHeight = ["clickhouse", "snowflake", "salesforce"].includes( - connector.name ?? "", - ) - ? "max-h-[38.5rem] min-h-[38.5rem]" - : "max-h-[34.5rem] min-h-[34.5rem]"; + this.formHeight = TALL_FORM_CONNECTORS.has(connector.name ?? "") + ? FORM_HEIGHT_TALL + : FORM_HEIGHT_DEFAULT; // IDs this.paramsFormId = `add-data-${connector.name}-form`; diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index be7ec4c6049..c6d4b2876e4 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -69,3 +69,11 @@ export const ALL_CONNECTORS = [...SOURCES, ...OLAP_ENGINES]; // Connectors that support multi-step forms (connector -> source) export const MULTI_STEP_CONNECTORS = ["gcs"]; + +export const FORM_HEIGHT_TALL = "max-h-[38.5rem] min-h-[38.5rem]"; +export const FORM_HEIGHT_DEFAULT = "max-h-[34.5rem] min-h-[34.5rem]"; +export const TALL_FORM_CONNECTORS = new Set([ + "clickhouse", + "snowflake", + "salesforce", +]); From 841025f20e0eeabe0235960fa9aea8f995aed9f7 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 14 Nov 2025 13:16:34 +0800 Subject: [PATCH 51/98] feedback --- .../features/sources/modal/AddDataForm.svelte | 30 +++++++++++-------- ...ction.svelte => AddDataFormSection.svelte} | 0 2 files changed, 17 insertions(+), 13 deletions(-) rename web-common/src/features/sources/modal/{FormSection.svelte => AddDataFormSection.svelte} (100%) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index d0e703d767c..a4c94e31555 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -25,7 +25,7 @@ import GCSMultiStepForm from "./GCSMultiStepForm.svelte"; import { AddDataFormManager } from "./AddDataFormManager"; import { hasOnlyDsn } from "./helpers"; - import FormSection from "./FormSection.svelte"; + import AddDataFormSection from "./AddDataFormSection.svelte"; export let connector: V1ConnectorDriver; export let formType: AddDataFormType; @@ -318,7 +318,7 @@ disableMarginTop > - - + - - + {:else if isConnectorForm && connector.configProperties?.some((property) => property.key === "dsn")} - + - + {:else if isMultiStepConnector} {#if stepState.step === "connector"} - - + {:else} - - + {/if} {:else} - - + {/if}
diff --git a/web-common/src/features/sources/modal/FormSection.svelte b/web-common/src/features/sources/modal/AddDataFormSection.svelte similarity index 100% rename from web-common/src/features/sources/modal/FormSection.svelte rename to web-common/src/features/sources/modal/AddDataFormSection.svelte From db50d47d0bcb512b3b57d49394fd0c041cf60018 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 14 Nov 2025 13:52:12 +0800 Subject: [PATCH 52/98] types, remove any --- .../sources/modal/AddDataFormManager.ts | 81 +++++++++++++------ 1 file changed, 58 insertions(+), 23 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 536e6418e8e..22af8740d3b 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -1,5 +1,10 @@ import { superForm, defaults } from "sveltekit-superforms"; -import { yup } from "sveltekit-superforms/adapters"; +import type { SuperValidated } from "sveltekit-superforms"; +import { + yup, + type Infer as YupInfer, + type InferIn as YupInferIn, +} from "sveltekit-superforms/adapters"; import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; import type { AddDataFormType } from "./types"; import { getValidationSchemaForConnector, dsnSchema } from "./FormValidation"; @@ -30,6 +35,17 @@ import type { ConnectorDriverProperty } from "@rilldata/web-common/runtime-clien import type { ClickHouseConnectorType } from "./constants"; import { applyClickHouseCloudRequirements } from "./helpers"; +// Minimal onUpdate event type carrying Superforms's validated form +type SuperFormUpdateEvent = { + form: SuperValidated, any, Record>; +}; + +// Shape of the step store for multi-step connectors +type ConnectorStepState = { + step: "connector" | "source"; + connectorConfig: Record | null; +}; + export class AddDataFormManager { formHeight: string; paramsFormId: string; @@ -55,8 +71,8 @@ export class AddDataFormManager { constructor(args: { connector: V1ConnectorDriver; formType: AddDataFormType; - onParamsUpdate: any; - onDsnUpdate: any; + onParamsUpdate: (event: SuperFormUpdateEvent) => void; + onDsnUpdate: (event: SuperFormUpdateEvent) => void; }) { const { connector, formType, onParamsUpdate, onDsnUpdate } = args; this.connector = connector; @@ -108,27 +124,36 @@ export class AddDataFormManager { ); // Superforms: params - const schema = yup( - getValidationSchemaForConnector(connector.name as string), + const paramsSchemaDef = getValidationSchemaForConnector( + connector.name as string, ); + const paramsAdapter = yup(paramsSchemaDef); + type ParamsOut = YupInfer; + type ParamsIn = YupInferIn; const initialFormValues = getInitialFormValuesFromProperties( this.properties, ); - this.params = superForm(initialFormValues, { + const paramsDefaults = defaults( + initialFormValues as Partial, + paramsAdapter, + ); + this.params = superForm(paramsDefaults, { SPA: true, - validators: schema, + validators: paramsAdapter, onUpdate: onParamsUpdate, resetForm: false, - } as any); + }); // Superforms: dsn - const dsnYupSchema = yup(dsnSchema); - this.dsn = superForm(defaults(dsnYupSchema), { + const dsnAdapter = yup(dsnSchema); + type DsnOut = YupInfer; + type DsnIn = YupInferIn; + this.dsn = superForm(defaults(dsnAdapter), { SPA: true, - validators: dsnYupSchema, + validators: dsnAdapter, onUpdate: onDsnUpdate, resetForm: false, - } as any); + }); } get isSourceForm(): boolean { @@ -154,14 +179,14 @@ export class AddDataFormManager { } handleSkip(): void { - const stepState = get(connectorStepStore) as any; + const stepState = get(connectorStepStore) as ConnectorStepState; if (!this.isMultiStepConnector || stepState.step !== "connector") return; - setConnectorConfig(get(this.params.form) as any); + setConnectorConfig(get(this.params.form) as Record); setStep("source"); } handleBack(onBack: () => void): void { - const stepState = get(connectorStepStore) as any; + const stepState = get(connectorStepStore) as ConnectorStepState; if (this.isMultiStepConnector && stepState.step === "source") { setStep("connector"); } else { @@ -227,13 +252,19 @@ export class AddDataFormManager { ); const isConnectorForm = this.formType === "connector"; - return async (event: any) => { + return async (event: { + form: SuperValidated< + Record, + any, + Record + >; + }) => { if (!event.form.valid) return; - const values = event.form.data as Record; + const values = event.form.data; try { - const stepState = get(connectorStepStore) as any; + const stepState = get(connectorStepStore) as ConnectorStepState; if (isMultiStepConnector && stepState.step === "source") { await submitAddSourceForm(queryClient, connector, values); onClose(); @@ -265,14 +296,14 @@ export class AddDataFormManager { onStringInputChange = ( event: Event, - taintedFields?: Record | null, + taintedFields?: Record | null, ) => { const target = event.target as HTMLInputElement; const { name, value } = target; if (name === "path") { const nameTainted = taintedFields && typeof taintedFields === "object" - ? Boolean((taintedFields as any)?.name) + ? Boolean(taintedFields?.name) : false; if (nameTainted) return; const inferred = inferSourceName(this.connector, value); @@ -302,11 +333,15 @@ export class AddDataFormManager { ); } return sanitized; - } catch (error: any) { + } catch (error: unknown) { if (error instanceof SyntaxError) { throw new Error(`Invalid JSON file: ${error.message}`); } - throw new Error(`Failed to read file: ${error.message}`); + const message = + error && typeof error === "object" && "message" in error + ? String((error as { message: unknown }).message) + : "Unknown error"; + throw new Error(`Failed to read file: ${message}`); } } @@ -318,7 +353,7 @@ export class AddDataFormManager { onlyDsn: boolean; filteredParamsProperties: ConnectorDriverProperty[]; filteredDsnProperties: ConnectorDriverProperty[]; - stepState: any; + stepState: ConnectorStepState | undefined; isMultiStepConnector: boolean; isConnectorForm: boolean; paramsFormValues: Record; From df8b5b0746951750fc2f885e99aa5c54cd1a974a Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 14 Nov 2025 13:57:35 +0800 Subject: [PATCH 53/98] remove helpers, move to utils --- .../features/sources/modal/AddDataForm.svelte | 2 +- .../sources/modal/AddDataFormManager.ts | 2 +- .../src/features/sources/modal/helpers.ts | 24 ---------------- .../src/features/sources/modal/utils.ts | 28 +++++++++++++++++-- 4 files changed, 28 insertions(+), 28 deletions(-) delete mode 100644 web-common/src/features/sources/modal/helpers.ts diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index a4c94e31555..e2c62153531 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -24,7 +24,7 @@ import YamlPreview from "./YamlPreview.svelte"; import GCSMultiStepForm from "./GCSMultiStepForm.svelte"; import { AddDataFormManager } from "./AddDataFormManager"; - import { hasOnlyDsn } from "./helpers"; + import { hasOnlyDsn } from "./utils"; import AddDataFormSection from "./AddDataFormSection.svelte"; export let connector: V1ConnectorDriver; diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 22af8740d3b..c81956ffeaa 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -33,7 +33,7 @@ import { compileConnectorYAML } from "../../connectors/code-utils"; import { compileSourceYAML, prepareSourceFormData } from "../sourceUtils"; import type { ConnectorDriverProperty } from "@rilldata/web-common/runtime-client"; import type { ClickHouseConnectorType } from "./constants"; -import { applyClickHouseCloudRequirements } from "./helpers"; +import { applyClickHouseCloudRequirements } from "./utils"; // Minimal onUpdate event type carrying Superforms's validated form type SuperFormUpdateEvent = { diff --git a/web-common/src/features/sources/modal/helpers.ts b/web-common/src/features/sources/modal/helpers.ts deleted file mode 100644 index 0829a2f5067..00000000000 --- a/web-common/src/features/sources/modal/helpers.ts +++ /dev/null @@ -1,24 +0,0 @@ -import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; -import type { ClickHouseConnectorType } from "./constants"; - -export function hasOnlyDsn( - connector: V1ConnectorDriver | undefined, - isConnectorForm: boolean, -): boolean { - if (!isConnectorForm) return false; - const props = connector?.configProperties ?? []; - const hasDsn = props.some((p) => p.key === "dsn"); - const hasOthers = props.some((p) => p.key !== "dsn"); - return hasDsn && !hasOthers; -} - -export function applyClickHouseCloudRequirements( - connectorName: string | undefined, - connectorType: ClickHouseConnectorType, - values: Record, -): Record { - if (connectorName === "clickhouse" && connectorType === "clickhouse-cloud") { - return { ...values, ssl: true, port: "8443" } as Record; - } - return values; -} diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 43d7dacf079..01b3527a61a 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -1,3 +1,7 @@ +import { humanReadableErrorMessage } from "../errors/errors"; +import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; +import type { ClickHouseConnectorType } from "./constants"; + export function isEmpty(val: any) { return ( val === undefined || @@ -17,8 +21,6 @@ export function normalizeErrors( return undefined; } -import { humanReadableErrorMessage } from "../errors/errors"; - export function normalizeConnectorError( connectorName: string, err: any, @@ -48,3 +50,25 @@ export function normalizeConnectorError( return { message, details }; } + +export function hasOnlyDsn( + connector: V1ConnectorDriver | undefined, + isConnectorForm: boolean, +): boolean { + if (!isConnectorForm) return false; + const props = connector?.configProperties ?? []; + const hasDsn = props.some((p) => p.key === "dsn"); + const hasOthers = props.some((p) => p.key !== "dsn"); + return hasDsn && !hasOthers; +} + +export function applyClickHouseCloudRequirements( + connectorName: string | undefined, + connectorType: ClickHouseConnectorType, + values: Record, +): Record { + if (connectorName === "clickhouse" && connectorType === "clickhouse-cloud") { + return { ...values, ssl: true, port: "8443" } as Record; + } + return values; +} From bcde4e20b50a04e6e46c5213519fb88721d7658d Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 14 Nov 2025 14:38:30 +0800 Subject: [PATCH 54/98] functon level comments --- .../src/features/sources/modal/utils.ts | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 01b3527a61a..260f7f3bef2 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -2,6 +2,10 @@ import { humanReadableErrorMessage } from "../errors/errors"; import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; import type { ClickHouseConnectorType } from "./constants"; +/** + * Returns true for undefined, null, empty string, or whitespace-only string. + * Useful for validating optional text inputs. + */ export function isEmpty(val: any) { return ( val === undefined || @@ -11,6 +15,13 @@ export function isEmpty(val: any) { ); } +/** + * Normalizes a variety of error shapes into a string, string[], or undefined. + * - If input is an array, returns it as-is. + * - If input is a string, returns it. + * - If input resembles a Zod `_errors` array, returns that. + * - Otherwise returns undefined. + */ export function normalizeErrors( err: any, ): string | string[] | null | undefined { @@ -21,6 +32,12 @@ export function normalizeErrors( return undefined; } +/** + * Converts unknown error inputs into a unified connector error shape. + * - Prefers native Error.message when present + * - Maps server error responses to human-readable messages via `humanReadableErrorMessage` + * - Returns `details` with original message when it differs from the human-readable message + */ export function normalizeConnectorError( connectorName: string, err: any, @@ -51,6 +68,10 @@ export function normalizeConnectorError( return { message, details }; } +/** + * Indicates whether a connector in "connector" mode exposes only a DSN field + * (i.e., DSN exists and no other config properties are present). + */ export function hasOnlyDsn( connector: V1ConnectorDriver | undefined, isConnectorForm: boolean, @@ -62,6 +83,11 @@ export function hasOnlyDsn( return hasDsn && !hasOthers; } +/** + * Applies ClickHouse Cloud-specific default requirements for connector values. + * - For ClickHouse Cloud: enforces `ssl: true` and `port: "8443"` + * - Otherwise returns values unchanged + */ export function applyClickHouseCloudRequirements( connectorName: string | undefined, connectorType: ClickHouseConnectorType, From 4055750e86b9cc1f3252f1591b4307557f043b9e Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 14 Nov 2025 14:52:05 +0800 Subject: [PATCH 55/98] lint --- web-common/src/features/sources/modal/AddDataForm.svelte | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index e2c62153531..1ef6f6d7c70 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -282,7 +282,10 @@ } function onStringInputChange(event: Event) { - formManager.onStringInputChange(event, $paramsTainted); + formManager.onStringInputChange( + event, + $paramsTainted as Record | null | undefined, + ); } From c1b6393fdbf1f2d9c3979d91cee34ec7b50d4848 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Mon, 17 Nov 2025 15:20:10 +0800 Subject: [PATCH 56/98] fix rebase --- web-common/src/features/sources/modal/constants.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index c6d4b2876e4..4bffc52825d 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -45,7 +45,6 @@ export const SOURCES = [ "azure", "bigquery", "gcs", - "postgres", "mysql", "postgres", "redshift", From 96dc09594273edebf5fbc09ded685429ad20c442 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Mon, 17 Nov 2025 15:32:46 +0800 Subject: [PATCH 57/98] undefined check for file path util --- web-common/src/features/sources/sourceUtils.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/web-common/src/features/sources/sourceUtils.ts b/web-common/src/features/sources/sourceUtils.ts index 4507ed39791..3431353956f 100644 --- a/web-common/src/features/sources/sourceUtils.ts +++ b/web-common/src/features/sources/sourceUtils.ts @@ -83,17 +83,18 @@ export function compileLocalFileSourceYAML(path: string) { return `${SOURCE_MODEL_FILE_TOP}\n\nconnector: duckdb\nsql: "${buildDuckDbQuery(path)}"`; } -function buildDuckDbQuery(path: string): string { - const extension = extractFileExtension(path); +function buildDuckDbQuery(path: string | undefined): string { + const safePath = typeof path === "string" ? path : ""; + const extension = extractFileExtension(safePath); if (extensionContainsParts(extension, [".csv", ".tsv", ".txt"])) { - return `select * from read_csv('${path}', auto_detect=true, ignore_errors=1, header=true)`; + return `select * from read_csv('${safePath}', auto_detect=true, ignore_errors=1, header=true)`; } else if (extensionContainsParts(extension, [".parquet"])) { - return `select * from read_parquet('${path}')`; + return `select * from read_parquet('${safePath}')`; } else if (extensionContainsParts(extension, [".json", ".ndjson"])) { - return `select * from read_json('${path}', auto_detect=true, format='auto')`; + return `select * from read_json('${safePath}', auto_detect=true, format='auto')`; } - return `select * from '${path}'`; + return `select * from '${safePath}'`; } /** From 70cccf57f3e93138320a0d0a6ba0e53903be246d Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 18 Nov 2025 01:06:56 +0800 Subject: [PATCH 58/98] step 3 data explorer scaffolding --- .../features/sources/modal/AddDataForm.svelte | 54 +++++++++++-------- .../sources/modal/AddDataFormManager.ts | 8 ++- .../sources/modal/DataExplorer.svelte | 11 ++++ .../sources/modal/connectorStepStore.ts | 2 +- 4 files changed, 49 insertions(+), 26 deletions(-) create mode 100644 web-common/src/features/sources/modal/DataExplorer.svelte diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 1ef6f6d7c70..420b431890e 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -26,6 +26,7 @@ import { AddDataFormManager } from "./AddDataFormManager"; import { hasOnlyDsn } from "./utils"; import AddDataFormSection from "./AddDataFormSection.svelte"; + import DataExplorer from "./DataExplorer.svelte"; export let connector: V1ConnectorDriver; export let formType: AddDataFormType; @@ -382,7 +383,7 @@ {handleFileUpload} /> - {:else} + {:else if stepState.step === "source"} + {:else} + +
+ +
{/if} {:else} {/if} - + {#if !(isMultiStepConnector && stepState.step === "explorer")} + + {/if}
diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index c81956ffeaa..ddf1e4fd0f0 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -42,7 +42,7 @@ type SuperFormUpdateEvent = { // Shape of the step store for multi-step connectors type ConnectorStepState = { - step: "connector" | "source"; + step: "connector" | "source" | "explorer"; connectorConfig: Record | null; }; @@ -189,6 +189,8 @@ export class AddDataFormManager { const stepState = get(connectorStepStore) as ConnectorStepState; if (this.isMultiStepConnector && stepState.step === "source") { setStep("connector"); + } else if (this.isMultiStepConnector && stepState.step === "explorer") { + setStep("source"); } else { onBack(); } @@ -267,7 +269,9 @@ export class AddDataFormManager { const stepState = get(connectorStepStore) as ConnectorStepState; if (isMultiStepConnector && stepState.step === "source") { await submitAddSourceForm(queryClient, connector, values); - onClose(); + // Advance to Step 3 (Data Explorer) instead of closing + setStep("explorer"); + return; } else if (isMultiStepConnector && stepState.step === "connector") { await submitAddConnectorForm(queryClient, connector, values, true); setConnectorConfig(values); diff --git a/web-common/src/features/sources/modal/DataExplorer.svelte b/web-common/src/features/sources/modal/DataExplorer.svelte new file mode 100644 index 00000000000..a904ef626bd --- /dev/null +++ b/web-common/src/features/sources/modal/DataExplorer.svelte @@ -0,0 +1,11 @@ + + +
+

Data Explorer

+

+ This is a placeholder for the Data Explorer step. +

+
diff --git a/web-common/src/features/sources/modal/connectorStepStore.ts b/web-common/src/features/sources/modal/connectorStepStore.ts index 9ce2451f126..7fd4399a77b 100644 --- a/web-common/src/features/sources/modal/connectorStepStore.ts +++ b/web-common/src/features/sources/modal/connectorStepStore.ts @@ -1,6 +1,6 @@ import { writable } from "svelte/store"; -export type ConnectorStep = "connector" | "source"; +export type ConnectorStep = "connector" | "source" | "explorer"; export const connectorStepStore = writable<{ step: ConnectorStep; From e7c2f8c35d9fb9d72695023c08f3d39686ebbed3 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 18 Nov 2025 01:15:40 +0800 Subject: [PATCH 59/98] table explorer form wip --- .../features/sources/modal/AddDataForm.svelte | 34 ++++++++++--------- .../sources/modal/DataExplorer.svelte | 11 ------ .../sources/modal/TableExplorerForm.svelte | 4 +++ 3 files changed, 22 insertions(+), 27 deletions(-) delete mode 100644 web-common/src/features/sources/modal/DataExplorer.svelte create mode 100644 web-common/src/features/sources/modal/TableExplorerForm.svelte diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 420b431890e..e974c6b9d84 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -26,7 +26,7 @@ import { AddDataFormManager } from "./AddDataFormManager"; import { hasOnlyDsn } from "./utils"; import AddDataFormSection from "./AddDataFormSection.svelte"; - import DataExplorer from "./DataExplorer.svelte"; + import TableExplorerForm from "./TableExplorerForm.svelte"; export let connector: V1ConnectorDriver; export let formType: AddDataFormType; @@ -400,9 +400,7 @@ {:else} -
- -
+ {/if} {:else} {/if} - - - + {#if !(isMultiStepConnector && stepState.step === "explorer")} + + {/if} + + {#if !(isMultiStepConnector && stepState.step === "explorer")} + + {/if} diff --git a/web-common/src/features/sources/modal/DataExplorer.svelte b/web-common/src/features/sources/modal/DataExplorer.svelte deleted file mode 100644 index a904ef626bd..00000000000 --- a/web-common/src/features/sources/modal/DataExplorer.svelte +++ /dev/null @@ -1,11 +0,0 @@ - - -
-

Data Explorer

-

- This is a placeholder for the Data Explorer step. -

-
diff --git a/web-common/src/features/sources/modal/TableExplorerForm.svelte b/web-common/src/features/sources/modal/TableExplorerForm.svelte new file mode 100644 index 00000000000..1336c559f53 --- /dev/null +++ b/web-common/src/features/sources/modal/TableExplorerForm.svelte @@ -0,0 +1,4 @@ + + +

Table Explorer

From ebf9582181d94dca13903ac9fbd92273104a00e6 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 18 Nov 2025 01:17:29 +0800 Subject: [PATCH 60/98] hide side panel from table explorer form --- .../features/sources/modal/AddDataForm.svelte | 38 +++++++++---------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index e974c6b9d84..93300d89adb 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -475,23 +475,23 @@ -
- {#if dsnError || paramsError || clickhouseError} - - {/if} + {#if !(isMultiStepConnector && stepState.step === "explorer")} +
+ {#if dsnError || paramsError || clickhouseError} + + {/if} - {#if !(isMultiStepConnector && stepState.step === "explorer")} - {/if} - {#if !(isMultiStepConnector && stepState.step === "explorer")} - {/if} -
+
+ {/if} From 3bf716e91a5d301cb25aeaefd424a64002b2c121 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 18 Nov 2025 01:23:29 +0800 Subject: [PATCH 61/98] reuse connector explorer in the form --- .../sources/modal/TableExplorerForm.svelte | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/web-common/src/features/sources/modal/TableExplorerForm.svelte b/web-common/src/features/sources/modal/TableExplorerForm.svelte index 1336c559f53..0fce2b4cb01 100644 --- a/web-common/src/features/sources/modal/TableExplorerForm.svelte +++ b/web-common/src/features/sources/modal/TableExplorerForm.svelte @@ -1,4 +1,38 @@ -

Table Explorer

+
+
+

Table explorer

+

+ Pick a table to power your first dashboard +

+
+ +
+ + + + +
+ +
+ + + +
From a32d20c018b18d03cc1a3732a541cccf30a3e0db Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 18 Nov 2025 01:34:04 +0800 Subject: [PATCH 62/98] generate model with AI --- .../features/sources/modal/AddDataForm.svelte | 49 ++++++++++++++++++- .../sources/modal/TableExplorerForm.svelte | 25 +++++++++- 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 93300d89adb..5b6cd018a16 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -27,6 +27,8 @@ import { hasOnlyDsn } from "./utils"; import AddDataFormSection from "./AddDataFormSection.svelte"; import TableExplorerForm from "./TableExplorerForm.svelte"; + import { goto } from "$app/navigation"; + import { createSqlModelFromTable } from "../../connectors/code-utils"; export let connector: V1ConnectorDriver; export let formType: AddDataFormType; @@ -122,6 +124,12 @@ let clickhouseParamsForm; let clickhouseDsnForm; let clickhouseShowSaveAnyway: boolean = false; + // Step 3 selection + let selectedConnectorForModel = ""; + let selectedDatabaseForModel = ""; + let selectedSchemaForModel = ""; + let selectedTableForModel = ""; + let generatingModel = false; $: isSubmitDisabled = (() => { if (onlyDsn || connectionTab === "dsn") { @@ -288,6 +296,28 @@ $paramsTainted as Record | null | undefined, ); } + + async function handleGenerateModelWithAI() { + if (!selectedConnectorForModel || !selectedTableForModel) return; + try { + generatingModel = true; + const addDevLimit = false; + const [newModelPath] = await createSqlModelFromTable( + queryClient, + selectedConnectorForModel, + selectedDatabaseForModel, + selectedSchemaForModel, + selectedTableForModel, + addDevLimit, + ); + await goto(`/files${newModelPath}`); + onClose(); + } catch (_e) { + // Swallow for now; future: surface error toast + } finally { + generatingModel = false; + } + }
@@ -400,7 +430,14 @@ {:else} - + { + selectedConnectorForModel = e.detail.connector; + selectedDatabaseForModel = e.detail.database; + selectedSchemaForModel = e.detail.schema; + selectedTableForModel = e.detail.table; + }} + /> {/if} {:else} + {:else} + {/if}
diff --git a/web-common/src/features/sources/modal/TableExplorerForm.svelte b/web-common/src/features/sources/modal/TableExplorerForm.svelte index 0fce2b4cb01..75bdaf09dfb 100644 --- a/web-common/src/features/sources/modal/TableExplorerForm.svelte +++ b/web-common/src/features/sources/modal/TableExplorerForm.svelte @@ -1,5 +1,6 @@ @@ -33,6 +55,5 @@
- From 2e9c083ee639f8748cfdd718f41da467cc6fc68e Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 18 Nov 2025 01:45:15 +0800 Subject: [PATCH 63/98] support table explorer --- .../features/sources/modal/AddDataForm.svelte | 28 +++++++++---------- .../sources/modal/AddDataFormManager.ts | 12 ++++++++ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 5b6cd018a16..1c139799d59 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -328,7 +328,17 @@
- {#if connector.name === "clickhouse"} + {#if stepState.step === "explorer"} + + { + selectedConnectorForModel = e.detail.connector; + selectedDatabaseForModel = e.detail.database; + selectedSchemaForModel = e.detail.schema; + selectedTableForModel = e.detail.table; + }} + /> + {:else if connector.name === "clickhouse"} - {:else if stepState.step === "source"} + {:else} - {:else} - - { - selectedConnectorForModel = e.detail.connector; - selectedDatabaseForModel = e.detail.database; - selectedSchemaForModel = e.detail.schema; - selectedTableForModel = e.detail.table; - }} - /> {/if} {:else} {/if} - {#if !(isMultiStepConnector && stepState.step === "explorer")} + {#if stepState.step !== "explorer"}
-
- - - - -
-
From b574d5359e7de0cbc6d5745edc654b9339fecd42 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 18 Nov 2025 18:27:52 +0800 Subject: [PATCH 68/98] remove --- web-common/src/features/sources/modal/TableExplorerForm.svelte | 1 - 1 file changed, 1 deletion(-) diff --git a/web-common/src/features/sources/modal/TableExplorerForm.svelte b/web-common/src/features/sources/modal/TableExplorerForm.svelte index 035449a7343..9ec0d270b39 100644 --- a/web-common/src/features/sources/modal/TableExplorerForm.svelte +++ b/web-common/src/features/sources/modal/TableExplorerForm.svelte @@ -1,5 +1,4 @@ @@ -524,13 +522,13 @@ {:else} {/if}
From 68454734a6c1cc2f034f3cd9836a4388d87a1d02 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 19 Nov 2025 14:42:36 +0800 Subject: [PATCH 71/98] type check --- .../features/connectors/explorer/connector-explorer-store.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-common/src/features/connectors/explorer/connector-explorer-store.ts b/web-common/src/features/connectors/explorer/connector-explorer-store.ts index 6642e98e0d1..97d350bccf3 100644 --- a/web-common/src/features/connectors/explorer/connector-explorer-store.ts +++ b/web-common/src/features/connectors/explorer/connector-explorer-store.ts @@ -54,7 +54,7 @@ export class ConnectorExplorerStore { expandedItems, selectedKey, }) - : writable({ showConnectors, expandedItems }); + : writable({ showConnectors, expandedItems, selectedKey }); } createItemIfNotExists( From ab97e56fb2adadf75786c80c75ea3c6cb4287c82 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 20 Nov 2025 17:54:54 +0800 Subject: [PATCH 72/98] clean up --- .../src/features/sources/modal/TableExplorerForm.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web-common/src/features/sources/modal/TableExplorerForm.svelte b/web-common/src/features/sources/modal/TableExplorerForm.svelte index 6d4a4d3c892..43a73df30f8 100644 --- a/web-common/src/features/sources/modal/TableExplorerForm.svelte +++ b/web-common/src/features/sources/modal/TableExplorerForm.svelte @@ -14,8 +14,8 @@ const store = connectorExplorerStore.duplicateStore( (connector, database = "", schema = "", table = "") => { // Only emit selection when a table is toggled - if (table) { - onSelect?.({ + if (table && onSelect) { + onSelect({ connector, database, schema, From 1d4d8ac2b1d347ca024aeb6f7d5e271891c1f1ba Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 20 Nov 2025 21:17:28 +0800 Subject: [PATCH 73/98] filter by connector in connector explorer --- .../features/connectors/explorer/ConnectorExplorer.svelte | 8 +++++++- web-common/src/features/sources/modal/AddDataForm.svelte | 1 + .../src/features/sources/modal/TableExplorerForm.svelte | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/web-common/src/features/connectors/explorer/ConnectorExplorer.svelte b/web-common/src/features/connectors/explorer/ConnectorExplorer.svelte index fd0fffcc1c6..4d593acd946 100644 --- a/web-common/src/features/connectors/explorer/ConnectorExplorer.svelte +++ b/web-common/src/features/connectors/explorer/ConnectorExplorer.svelte @@ -8,6 +8,7 @@ export let store: ConnectorExplorerStore; export let olapOnly: boolean = false; + export let limitToConnector: string | undefined = undefined; $: ({ instanceId } = $runtime); @@ -17,11 +18,16 @@ select: (data) => { if (!data?.connectors) return; - const filtered = ( + let filtered = ( olapOnly ? data.connectors.filter((c) => c?.driver?.implementsOlap) : data.connectors ).sort((a, b) => (a?.name as string).localeCompare(b?.name as string)); + if (limitToConnector) { + filtered = filtered.filter( + (c) => (c?.name as string) === limitToConnector, + ); + } return { connectors: filtered }; }, }, diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index d798bd6b440..47b6c1bc7d5 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -345,6 +345,7 @@ {#if stepState.step === "explorer"} { selectedConnectorForModel = detail.connector; selectedDatabaseForModel = detail.database; diff --git a/web-common/src/features/sources/modal/TableExplorerForm.svelte b/web-common/src/features/sources/modal/TableExplorerForm.svelte index 43a73df30f8..73e18ce159f 100644 --- a/web-common/src/features/sources/modal/TableExplorerForm.svelte +++ b/web-common/src/features/sources/modal/TableExplorerForm.svelte @@ -2,6 +2,7 @@ import ConnectorExplorer from "../../connectors/explorer/ConnectorExplorer.svelte"; import { connectorExplorerStore } from "../../connectors/explorer/connector-explorer-store"; + export let connectorName: string | undefined = undefined; export let onSelect: | ((detail: { connector: string; @@ -36,5 +37,5 @@
- + From c3e4818f54c21b1d3cb5d86fbf6e7fd26da580b9 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 20 Nov 2025 21:40:54 +0800 Subject: [PATCH 74/98] enabled perf --- .../explorer/ConnectorExplorer.svelte | 25 ++++++++++++------- .../features/sources/modal/AddDataForm.svelte | 2 +- .../sources/modal/TableExplorerForm.svelte | 5 ++-- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/web-common/src/features/connectors/explorer/ConnectorExplorer.svelte b/web-common/src/features/connectors/explorer/ConnectorExplorer.svelte index 4d593acd946..dca557d511c 100644 --- a/web-common/src/features/connectors/explorer/ConnectorExplorer.svelte +++ b/web-common/src/features/connectors/explorer/ConnectorExplorer.svelte @@ -5,15 +5,17 @@ import { runtime } from "../../../runtime-client/runtime-store"; import ConnectorEntry from "./ConnectorEntry.svelte"; import type { ConnectorExplorerStore } from "./connector-explorer-store"; + import type { V1ConnectorDriver } from "../../../runtime-client"; export let store: ConnectorExplorerStore; export let olapOnly: boolean = false; - export let limitToConnector: string | undefined = undefined; + export let limitedConnectorDriver: V1ConnectorDriver | undefined = undefined; $: ({ instanceId } = $runtime); $: connectors = createRuntimeServiceAnalyzeConnectors(instanceId, { query: { + enabled: !!instanceId && !limitedConnectorDriver, // sort alphabetically select: (data) => { if (!data?.connectors) return; @@ -23,16 +25,21 @@ ? data.connectors.filter((c) => c?.driver?.implementsOlap) : data.connectors ).sort((a, b) => (a?.name as string).localeCompare(b?.name as string)); - if (limitToConnector) { - filtered = filtered.filter( - (c) => (c?.name as string) === limitToConnector, - ); - } return { connectors: filtered }; }, }, }); $: ({ data, error } = $connectors); + $: connectorsData = limitedConnectorDriver + ? { + connectors: [ + { + name: (limitedConnectorDriver.name as string) ?? "", + driver: limitedConnectorDriver, + }, + ], + } + : data;
@@ -40,12 +47,12 @@ {error.message} - {:else if data?.connectors} - {#if data.connectors.length === 0} + {:else if connectorsData?.connectors} + {#if connectorsData.connectors.length === 0} No data found. Add data to get started! {:else}
    - {#each data.connectors as connector (connector.name)} + {#each connectorsData.connectors as connector (connector.name)} {/each}
diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 47b6c1bc7d5..32d76221283 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -345,7 +345,7 @@ {#if stepState.step === "explorer"} { selectedConnectorForModel = detail.connector; selectedDatabaseForModel = detail.database; diff --git a/web-common/src/features/sources/modal/TableExplorerForm.svelte b/web-common/src/features/sources/modal/TableExplorerForm.svelte index 73e18ce159f..19ef278b8a5 100644 --- a/web-common/src/features/sources/modal/TableExplorerForm.svelte +++ b/web-common/src/features/sources/modal/TableExplorerForm.svelte @@ -1,8 +1,9 @@ -
-
-

Data explorer

-

Pick a table to power your model

-
+
+ + + + +
+
+

Data explorer

+

Pick a table to power your model

+
-
+
- -
+ +
+
From 52ac4eba9d585c759d7453c2a5700cbac9f9254c Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 21 Nov 2025 17:23:52 +0800 Subject: [PATCH 82/98] copy change --- .../sources/modal/DataExplorerDialog.svelte | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/web-common/src/features/sources/modal/DataExplorerDialog.svelte b/web-common/src/features/sources/modal/DataExplorerDialog.svelte index e4c945ec418..f28662729e4 100644 --- a/web-common/src/features/sources/modal/DataExplorerDialog.svelte +++ b/web-common/src/features/sources/modal/DataExplorerDialog.svelte @@ -51,9 +51,19 @@ class="w-full md:w-64 border-b md:border-b-0 md:border-r border-gray-200" >
-

- {connectorDriver?.name || "Connector"} -

+ {#if sidebar.length > 0} +

Existing connectors

+

+ Choose data from an existing connector create a new one +

+ {:else} +

+ Connected successfully! +

+

+ Pick a table to power your first model +

+ {/if}
{#if sidebar.length > 0} {#each sidebar as c (c.name)} From 98faed28e0ee6764ac9dbbe11fd5a1557f406ae3 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 21 Nov 2025 17:51:55 +0800 Subject: [PATCH 83/98] layout, styles --- .../src/features/sources/modal/AddDataForm.svelte | 7 ++++++- .../sources/modal/DataExplorerDialog.svelte | 14 +++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 446cbe105fb..d24f8f859fd 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -34,6 +34,7 @@ } from "../../connectors/code-utils"; import { runtime } from "@rilldata/web-common/runtime-client/runtime-store"; import { useIsModelingSupportedForConnectorOLAP as useIsModelingSupportedForConnector } from "../../connectors/selectors"; + import { cn } from "@rilldata/web-common/lib/shadcn"; export let connector: V1ConnectorDriver; export let formType: AddDataFormType; @@ -340,7 +341,11 @@ class="add-data-form-panel flex-1 flex flex-col min-w-0 md:pr-0 pr-0 relative" >
{#if stepState.step === "explorer"} diff --git a/web-common/src/features/sources/modal/DataExplorerDialog.svelte b/web-common/src/features/sources/modal/DataExplorerDialog.svelte index f28662729e4..984e5d6aaa2 100644 --- a/web-common/src/features/sources/modal/DataExplorerDialog.svelte +++ b/web-common/src/features/sources/modal/DataExplorerDialog.svelte @@ -48,20 +48,20 @@
-
+

Data explorer

Pick a table to power your model

-
- - +
+ +
From ea4d5e102ec3ce4989d678baa86f155fc5cd07c6 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 21 Nov 2025 18:10:21 +0800 Subject: [PATCH 84/98] overflow y --- web-common/src/features/sources/modal/DataExplorerDialog.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-common/src/features/sources/modal/DataExplorerDialog.svelte b/web-common/src/features/sources/modal/DataExplorerDialog.svelte index 984e5d6aaa2..2a0594ee1ff 100644 --- a/web-common/src/features/sources/modal/DataExplorerDialog.svelte +++ b/web-common/src/features/sources/modal/DataExplorerDialog.svelte @@ -90,7 +90,7 @@

Pick a table to power your model

-
+
From 9ee22aeb96b5243171ca03686e4f79e349a1461a Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 21 Nov 2025 23:02:59 +0800 Subject: [PATCH 85/98] sticky leftsidebar container, styles --- web-common/src/features/sources/modal/AddDataForm.svelte | 6 ++++-- .../src/features/sources/modal/DataExplorerDialog.svelte | 8 +++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index d24f8f859fd..79109616855 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -342,9 +342,11 @@ >
{#if stepState.step === "explorer"} diff --git a/web-common/src/features/sources/modal/DataExplorerDialog.svelte b/web-common/src/features/sources/modal/DataExplorerDialog.svelte index 2a0594ee1ff..6cfa7b30856 100644 --- a/web-common/src/features/sources/modal/DataExplorerDialog.svelte +++ b/web-common/src/features/sources/modal/DataExplorerDialog.svelte @@ -48,9 +48,9 @@