diff --git a/apps/design-system/README.md b/apps/design-system/README.md index 96a8eaf8fefd0..127060277616e 100644 --- a/apps/design-system/README.md +++ b/apps/design-system/README.md @@ -47,3 +47,26 @@ Open [http://localhost:3003](http://localhost:3003) in your browser to see the r ### Watching for MDX changes The `dev:full` command automatically watches for changes to MDX files with hot reload. If you're running the `pnpm dev` separately, you'll need to run `pnpm content:dev` in a separate terminal shell to watch for content changes. + +### Adding components + +The design system _references_ components rather than housing them. That’s an important distinction to make, as everything that follows here is about the documentation of components. You can add or edit components in one of these two places: + +- [`packages/ui`](https://github.com/supabase/supabase/tree/master/packages/ui): basic UI components +- [`packages/ui-patterns`](https://github.com/supabase/supabase/tree/master/packages/ui-patterns): components which are built using NPM libraries or amalgamations of components from `patterns/ui` + +With that out of the way, there are several parts of this design system that need to be manually updated after components have been added or removed (from documentation). These include: + +- `config/docs.ts`: list of components in the sidebar +- `content/docs`: where the actual component documentation `.mdx` file lives +- `registry/examples.ts`: list of example components +- `registry/default/example`: where the actual example component(s) live +- `registry/charts.ts`: Chart components +- `registry/fragments.ts`: Fragment components + +You will need to rebuild the design system’s registry each time a new example component is added. In other words: whenever a new file enters `registry`, it needs to be rebuilt. You can do that via: + +```bash +cd apps/design-system +pnpm build:registry +``` diff --git a/apps/design-system/__registry__/index.tsx b/apps/design-system/__registry__/index.tsx index 912c60fe77cd3..005d827880186 100644 --- a/apps/design-system/__registry__/index.tsx +++ b/apps/design-system/__registry__/index.tsx @@ -588,6 +588,50 @@ export const Index: Record = { subcategory: "undefined", chunks: [] }, + "data-input-demo": { + name: "data-input-demo", + type: "components:example", + registryDependencies: ["data-input"], + component: React.lazy(() => import("@/registry/default/example/data-input-demo")), + source: "", + files: ["registry/default/example/data-input-demo.tsx"], + category: "undefined", + subcategory: "undefined", + chunks: [] + }, + "data-input-with-copy": { + name: "data-input-with-copy", + type: "components:example", + registryDependencies: ["data-input"], + component: React.lazy(() => import("@/registry/default/example/data-input-with-copy")), + source: "", + files: ["registry/default/example/data-input-with-copy.tsx"], + category: "undefined", + subcategory: "undefined", + chunks: [] + }, + "data-input-with-copy-secret": { + name: "data-input-with-copy-secret", + type: "components:example", + registryDependencies: ["data-input"], + component: React.lazy(() => import("@/registry/default/example/data-input-with-copy-secret")), + source: "", + files: ["registry/default/example/data-input-with-copy-secret.tsx"], + category: "undefined", + subcategory: "undefined", + chunks: [] + }, + "data-input-with-reveal-copy": { + name: "data-input-with-reveal-copy", + type: "components:example", + registryDependencies: ["data-input"], + component: React.lazy(() => import("@/registry/default/example/data-input-with-reveal-copy")), + source: "", + files: ["registry/default/example/data-input-with-reveal-copy.tsx"], + category: "undefined", + subcategory: "undefined", + chunks: [] + }, "date-picker-demo": { name: "date-picker-demo", type: "components:example", diff --git a/apps/design-system/config/docs.ts b/apps/design-system/config/docs.ts index c41034a9f2f22..35407e0b4333c 100644 --- a/apps/design-system/config/docs.ts +++ b/apps/design-system/config/docs.ts @@ -117,6 +117,11 @@ export const docsConfig: DocsConfig = { href: '/docs/fragments/inner-side-menu', items: [], }, + { + title: 'Data Input', + href: '/docs/fragments/data-input', + items: [], + }, { title: 'Form Item Layout', href: '/docs/fragments/form-item-layout', diff --git a/apps/design-system/content/docs/fragments/data-input.mdx b/apps/design-system/content/docs/fragments/data-input.mdx new file mode 100644 index 0000000000000..040c3e9dcc5ee --- /dev/null +++ b/apps/design-system/content/docs/fragments/data-input.mdx @@ -0,0 +1,47 @@ +--- +title: Data Input +description: Set, read, or copy a value on a single line. +component: true +--- + + + +Referred to as `Input` in code, not `DataInput`. Also not to be confused with the atomic [Input](../components/input) component. + +## Usage + +### Read-only values + +Input should be used for read-only values that can be copied or otherwise interacted with, as the input element is both keyboard and mouse-friendly. + + + +### Sensitive values + +Inputs with sensitive values can be both revealed _and_ copied, but only in succession. This reduces the amount of actions on screen, thus simplifying the interface. + + + +You can also partially truncate the value by overriding the placeholder value. + +Consider if the value needs to be revealed in the first place, as only copying is sufficient in most cases. + +A happy medium might be to display a partially masked value but saving the actual value in the clipboard. To do this, pass a pre-masked string as the `value` prop and override the `onCopy` callback to copy the real value. + + + +### Password managers + +In some cases, you may need to add the following attribute to your input to prevent password managers from applying their widgets and dropdowns to the input: + +```jsx + +``` diff --git a/apps/design-system/registry/default/example/data-input-demo.tsx b/apps/design-system/registry/default/example/data-input-demo.tsx new file mode 100644 index 0000000000000..c31326de17f35 --- /dev/null +++ b/apps/design-system/registry/default/example/data-input-demo.tsx @@ -0,0 +1,5 @@ +import { Input } from 'ui-patterns/DataInputs/Input' + +export default function DataInputDemo() { + return +} diff --git a/apps/design-system/registry/default/example/data-input-with-copy-secret.tsx b/apps/design-system/registry/default/example/data-input-with-copy-secret.tsx new file mode 100644 index 0000000000000..f09531f6a25a8 --- /dev/null +++ b/apps/design-system/registry/default/example/data-input-with-copy-secret.tsx @@ -0,0 +1,18 @@ +import { Input } from 'ui-patterns/DataInputs/Input' + +export default function DataInputWithCopySecret() { + const actualValue = 'sb_secret_1234567890' + const maskedValue = 'sb_secret_123•••••••' + + return ( + { + navigator.clipboard.writeText(actualValue) + }} + /> + ) +} diff --git a/apps/design-system/registry/default/example/data-input-with-copy.tsx b/apps/design-system/registry/default/example/data-input-with-copy.tsx new file mode 100644 index 0000000000000..52a3cc7bd766c --- /dev/null +++ b/apps/design-system/registry/default/example/data-input-with-copy.tsx @@ -0,0 +1,5 @@ +import { Input } from 'ui-patterns/DataInputs/Input' + +export default function DataInputWithCopy() { + return +} diff --git a/apps/design-system/registry/default/example/data-input-with-reveal-copy.tsx b/apps/design-system/registry/default/example/data-input-with-reveal-copy.tsx new file mode 100644 index 0000000000000..168a155961c9c --- /dev/null +++ b/apps/design-system/registry/default/example/data-input-with-reveal-copy.tsx @@ -0,0 +1,5 @@ +import { Input } from 'ui-patterns/DataInputs/Input' + +export default function DataInputWithRevealCopy() { + return +} diff --git a/apps/design-system/registry/examples.ts b/apps/design-system/registry/examples.ts index 4e389b1d238be..b47de24fed577 100644 --- a/apps/design-system/registry/examples.ts +++ b/apps/design-system/registry/examples.ts @@ -385,12 +385,30 @@ export const examples: Registry = [ registryDependencies: ['context-menu'], files: ['example/context-menu-demo.tsx'], }, - // { - // name: 'data-table-demo', - // type: 'components:example', - // registryDependencies: ['data-table'], - // files: ['example/data-table-demo.tsx'], - // }, + { + name: 'data-input-demo', + type: 'components:example', + registryDependencies: ['data-input'], + files: ['example/data-input-demo.tsx'], + }, + { + name: 'data-input-with-copy', + type: 'components:example', + registryDependencies: ['data-input'], + files: ['example/data-input-with-copy.tsx'], + }, + { + name: 'data-input-with-copy-secret', + type: 'components:example', + registryDependencies: ['data-input'], + files: ['example/data-input-with-copy-secret.tsx'], + }, + { + name: 'data-input-with-reveal-copy', + type: 'components:example', + registryDependencies: ['data-input'], + files: ['example/data-input-with-reveal-copy.tsx'], + }, { name: 'date-picker-demo', type: 'components:example', diff --git a/apps/studio/components/interfaces/Database/Replication/ComingSoon.tsx b/apps/studio/components/interfaces/Database/Replication/ComingSoon.tsx index c6e54a9877f64..44a41e88ecb74 100644 --- a/apps/studio/components/interfaces/Database/Replication/ComingSoon.tsx +++ b/apps/studio/components/interfaces/Database/Replication/ComingSoon.tsx @@ -96,7 +96,7 @@ const ReplicationStaticMockup = ({ projectRef }: { projectRef: string }) => { blank: BlankNode, cta: () => CTANode({ projectRef }), }), - [] + [projectRef] ) return ( diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/AdvancedSettings.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/AdvancedSettings.tsx new file mode 100644 index 0000000000000..4f3b864f379cb --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/AdvancedSettings.tsx @@ -0,0 +1,81 @@ +import { UseFormReturn } from 'react-hook-form' + +import { + Accordion_Shadcn_, + AccordionContent_Shadcn_, + AccordionItem_Shadcn_, + AccordionTrigger_Shadcn_, + FormControl_Shadcn_, + FormField_Shadcn_, + Input_Shadcn_, +} from 'ui' +import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout' +import { DestinationPanelSchemaType } from './DestinationPanel.schema' + +export const AdvancedSettings = ({ form }: { form: UseFormReturn }) => { + const { type } = form.watch() + + return ( + + + + Advanced Settings + + + ( + + + { + const val = e.target.value + field.onChange(val === '' ? undefined : Number(val)) + }} + placeholder="Leave empty for default" + /> + + + )} + /> + {type === 'BigQuery' && ( + ( + + + { + const val = e.target.value + field.onChange(val === '' ? undefined : Number(val)) + }} + placeholder="Leave empty for default" + /> + + + )} + /> + )} + + + + ) +} diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.constants.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.constants.ts new file mode 100644 index 0000000000000..c94f883cd0e1c --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.constants.ts @@ -0,0 +1,5 @@ +// Hardcoded value for `s3AccessKeyId` field in the form to indicate creating a new key +export const CREATE_NEW_KEY = 'create-new' + +// Hardcoded value for `namespace` field in the form to indicate creating a new namespace +export const CREATE_NEW_NAMESPACE = 'create-new-namespace' diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.schema.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.schema.ts new file mode 100644 index 0000000000000..4eaa16f39c7f1 --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.schema.ts @@ -0,0 +1,63 @@ +import * as z from 'zod' + +const types = ['BigQuery', 'Analytics Bucket'] as const +export const TypeEnum = z.enum(types) + +// [Joshen] JFYI if we plan to add another type here, I reckon we split this out into smaller components +// then as FormSchema is getting quite complex with some fields that aren't necessary if the type is one or the other +export const DestinationPanelFormSchema = z + .object({ + // Common fields + type: TypeEnum, + name: z.string().min(1, 'Name is required'), + publicationName: z.string().min(1, 'Publication is required'), + maxFillMs: z.number().min(1, 'Max Fill milliseconds should be greater than 0').int().optional(), + // BigQuery fields + projectId: z.string().optional(), + datasetId: z.string().optional(), + serviceAccountKey: z.string().optional(), + maxStalenessMins: z.number().nonnegative().optional(), + // Analytics Bucket fields, only warehouse name and namespace are visible + editable fields + warehouseName: z.string().optional(), + namespace: z.string().optional(), + newNamespaceName: z.string().optional(), + catalogToken: z.string().optional(), + s3AccessKeyId: z.string().optional(), + s3SecretAccessKey: z.string().optional(), + s3Region: z.string().optional(), + }) + .refine( + (data) => { + if (data.type === 'BigQuery') { + return ( + data.projectId && + data.projectId.length > 0 && + data.datasetId && + data.datasetId.length > 0 && + data.serviceAccountKey && + data.serviceAccountKey.length > 0 + ) + } else if (data.type === 'Analytics Bucket') { + const hasValidNamespace = + (data.namespace && data.namespace.length > 0) || + (data.namespace === 'create-new-namespace' && + data.newNamespaceName && + data.newNamespaceName.length > 0) + + return ( + data.warehouseName && + data.warehouseName.length > 0 && + hasValidNamespace && + data.s3Region && + data.s3Region.length > 0 + ) + } + return true + }, + { + message: 'All fields are required for the selected destination type', + path: ['projectId'], + } + ) + +export type DestinationPanelSchemaType = z.infer diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx similarity index 64% rename from apps/studio/components/interfaces/Database/Replication/DestinationPanel.tsx rename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx index 102a14e09fe13..a4ef25e9f17c5 100644 --- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel.tsx +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx @@ -1,11 +1,15 @@ import { zodResolver } from '@hookform/resolvers/zod' +import { snakeCase } from 'lodash' import { useEffect, useMemo, useState } from 'react' import { useForm } from 'react-hook-form' import { toast } from 'sonner' import * as z from 'zod' import { useParams } from 'common' +import { getCatalogURI } from 'components/interfaces/Storage/StorageSettings/StorageSettings.utils' import { InlineLink } from 'components/ui/InlineLink' +import { getKeys, useAPIKeysQuery } from 'data/api-keys/api-keys-query' +import { useProjectSettingsV2Query } from 'data/config/project-settings-v2-query' import { useCheckPrimaryKeysExists } from 'data/database/primary-keys-exists-query' import { useCreateDestinationPipelineMutation } from 'data/replication/create-destination-pipeline-mutation' import { useReplicationDestinationByIdQuery } from 'data/replication/destination-by-id-query' @@ -13,16 +17,14 @@ import { useReplicationPipelineByIdQuery } from 'data/replication/pipeline-by-id import { useReplicationPublicationsQuery } from 'data/replication/publications-query' import { useStartPipelineMutation } from 'data/replication/start-pipeline-mutation' import { useUpdateDestinationPipelineMutation } from 'data/replication/update-destination-pipeline-mutation' +import { useIcebergNamespaceCreateMutation } from 'data/storage/iceberg-namespace-create-mutation' +import { useS3AccessKeyCreateMutation } from 'data/storage/s3-access-key-create-mutation' import { useSelectedProjectQuery } from 'hooks/misc/useSelectedProject' import { PipelineStatusRequestStatus, usePipelineRequestStatus, } from 'state/replication-pipeline-request-status' import { - Accordion_Shadcn_, - AccordionContent_Shadcn_, - AccordionItem_Shadcn_, - AccordionTrigger_Shadcn_, Button, DialogSectionSeparator, Form_Shadcn_, @@ -41,28 +43,18 @@ import { SheetHeader, SheetSection, SheetTitle, - TextArea_Shadcn_, } from 'ui' import { Admonition } from 'ui-patterns' import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout' -import NewPublicationPanel from './NewPublicationPanel' -import PublicationsComboBox from './PublicationsComboBox' -import { ReplicationDisclaimerDialog } from './ReplicationDisclaimerDialog' +import { NewPublicationPanel } from '../NewPublicationPanel' +import { PublicationsComboBox } from '../PublicationsComboBox' +import { ReplicationDisclaimerDialog } from '../ReplicationDisclaimerDialog' +import { AdvancedSettings } from './AdvancedSettings' +import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationPanel.constants' +import { DestinationPanelFormSchema as FormSchema, TypeEnum } from './DestinationPanel.schema' +import { AnalyticsBucketFields, BigQueryFields } from './DestinationPanelFields' const formId = 'destination-editor' -const types = ['BigQuery'] as const -const TypeEnum = z.enum(types) - -const FormSchema = z.object({ - type: TypeEnum, - name: z.string().min(1, 'Name is required'), - projectId: z.string().min(1, 'Project id is required'), - datasetId: z.string().min(1, 'Dataset id is required'), - serviceAccountKey: z.string().min(1, 'Service account key is required'), - publicationName: z.string().min(1, 'Publication is required'), - maxFillMs: z.number().min(1, 'Max Fill milliseconds should be greater than 0').int().optional(), - maxStalenessMins: z.number().nonnegative().optional(), -}) interface DestinationPanelProps { visible: boolean @@ -93,6 +85,7 @@ export const DestinationPanel = ({ const [pendingFormValues, setPendingFormValues] = useState | null>( null ) + const [isFormInteracting, setIsFormInteracting] = useState(false) const { mutateAsync: createDestinationPipeline, isLoading: creatingDestinationPipeline } = useCreateDestinationPipelineMutation({ @@ -106,6 +99,12 @@ export const DestinationPanel = ({ const { mutateAsync: startPipeline, isLoading: startingPipeline } = useStartPipelineMutation() + const { mutateAsync: createS3AccessKey, isLoading: isCreatingS3AccessKey } = + useS3AccessKeyCreateMutation() + + const { mutateAsync: createNamespace, isLoading: isCreatingNamespace } = + useIcebergNamespaceCreateMutation() + const { data: publications = [], isLoading: isLoadingPublications, @@ -123,24 +122,44 @@ export const DestinationPanel = ({ pipelineId: existingDestination?.pipelineId, }) + const { data: apiKeys } = useAPIKeysQuery({ projectRef, reveal: true }) + const { serviceKey } = getKeys(apiKeys) + const catalogToken = serviceKey?.api_key ?? '' + + const { data: projectSettings } = useProjectSettingsV2Query({ projectRef }) + const defaultValues = useMemo(() => { - const bigQueryConfig = - destinationData && 'big_query' in destinationData.config - ? destinationData?.config.big_query - : null + // Type guards to safely access union properties + const config = destinationData?.config + const isBigQueryConfig = config && 'big_query' in config + const isIcebergConfig = config && 'iceberg' in config return { - type: TypeEnum.enum.BigQuery, + // Common fields + type: (isBigQueryConfig + ? TypeEnum.enum.BigQuery + : isIcebergConfig + ? TypeEnum.enum['Analytics Bucket'] + : TypeEnum.enum.BigQuery) as z.infer, name: destinationData?.name ?? '', - projectId: bigQueryConfig?.project_id ?? '', - datasetId: bigQueryConfig?.dataset_id ?? '', - // For now, the password will always be set as empty for security reasons. - serviceAccountKey: bigQueryConfig?.service_account_key ?? '', publicationName: pipelineData?.config.publication_name ?? '', maxFillMs: pipelineData?.config?.batch?.max_fill_ms, - maxStalenessMins: bigQueryConfig?.max_staleness_mins, + // BigQuery fields + projectId: isBigQueryConfig ? config.big_query.project_id : '', + datasetId: isBigQueryConfig ? config.big_query.dataset_id : '', + serviceAccountKey: isBigQueryConfig ? config.big_query.service_account_key : '', + maxStalenessMins: isBigQueryConfig ? config.big_query.max_staleness_mins : undefined, + // Analytics Bucket fields + warehouseName: isIcebergConfig ? config.iceberg.supabase.warehouse_name : '', + namespace: isIcebergConfig ? config.iceberg.supabase.namespace : '', + newNamespaceName: '', + catalogToken: isIcebergConfig ? config.iceberg.supabase.catalog_token : catalogToken, + s3AccessKeyId: isIcebergConfig ? config.iceberg.supabase.s3_access_key_id : '', + s3SecretAccessKey: isIcebergConfig ? config.iceberg.supabase.s3_secret_access_key : '', + s3Region: + projectSettings?.region ?? (isIcebergConfig ? config.iceberg.supabase.s3_region : ''), } - }, [destinationData, pipelineData]) + }, [destinationData, pipelineData, catalogToken, projectSettings]) const form = useForm>({ mode: 'onBlur', @@ -148,8 +167,13 @@ export const DestinationPanel = ({ resolver: zodResolver(FormSchema), defaultValues, }) - const publicationName = form.watch('publicationName') - const isSaving = creatingDestinationPipeline || updatingDestinationPipeline || startingPipeline + const { publicationName, type: selectedType, warehouseName } = form.watch() + const isSaving = + creatingDestinationPipeline || + updatingDestinationPipeline || + startingPipeline || + isCreatingS3AccessKey || + isCreatingNamespace const publicationNames = useMemo(() => publications?.map((pub) => pub.name) ?? [], [publications]) const isSelectedPublicationMissing = @@ -169,6 +193,35 @@ export const DestinationPanel = ({ const isSubmitDisabled = isSaving || isSelectedPublicationMissing || isLoadingCheck || hasTablesWithNoPrimaryKeys + // Helper function to handle namespace creation if needed + const resolveNamespace = async (data: z.infer) => { + if (data.namespace === CREATE_NEW_NAMESPACE) { + if (!data.newNamespaceName) { + throw new Error('New namespace name is required') + } + + // Construct catalog URI for namespace creation + const protocol = projectSettings?.app_config?.protocol ?? 'https' + const endpoint = + projectSettings?.app_config?.storage_endpoint || projectSettings?.app_config?.endpoint + const catalogUri = getCatalogURI(project?.ref ?? '', protocol, endpoint) + + await createNamespace({ + catalogUri, + warehouse: data.warehouseName!, + token: catalogToken, + namespace: data.newNamespaceName, + }) + + return data.newNamespaceName + } + return data.namespace + } + + // [Joshen] I reckon this function can be refactored to be a bit more modular, it's currently pretty + // complicated with 4 different types of flows -> edit bigquery/analytics, and create bigquery/analytics + // At first glance we could try grouping as edit / create bigquery, edit / create analytics + // since the destination config seems rather similar between edit and create for the same type const submitPipeline = async (data: z.infer) => { if (!projectRef) return console.error('Project ref is required') if (!sourceId) return console.error('Source id is required') @@ -180,13 +233,42 @@ export const DestinationPanel = ({ if (editMode && existingDestination) { if (!existingDestination.pipelineId) return console.error('Pipeline id is required') - const bigQueryConfig: any = { - projectId: data.projectId, - datasetId: data.datasetId, - serviceAccountKey: data.serviceAccountKey, - } - if (!!data.maxStalenessMins) { - bigQueryConfig.maxStalenessMins = data.maxStalenessMins + let destinationConfig: any = {} + + if (data.type === 'BigQuery') { + const bigQueryConfig: any = { + projectId: data.projectId, + datasetId: data.datasetId, + serviceAccountKey: data.serviceAccountKey, + } + if (!!data.maxStalenessMins) { + bigQueryConfig.maxStalenessMins = data.maxStalenessMins + } + destinationConfig = { bigQuery: bigQueryConfig } + } else if (data.type === 'Analytics Bucket') { + let s3Keys = { accessKey: data.s3AccessKeyId, secretKey: data.s3SecretAccessKey } + + if (data.s3AccessKeyId === CREATE_NEW_KEY) { + const newKeys = await createS3AccessKey({ + projectRef, + description: `Autogenerated key for replication to ${snakeCase(warehouseName)}`, + }) + s3Keys = { accessKey: newKeys.access_key, secretKey: newKeys.secret_key } + } + + // Resolve namespace (create if needed) + const finalNamespace = await resolveNamespace(data) + + const icebergConfig: any = { + projectRef: projectRef, + warehouseName: data.warehouseName, + namespace: finalNamespace, + catalogToken: data.catalogToken, + s3AccessKeyId: s3Keys.accessKey, + s3SecretAccessKey: s3Keys.secretKey, + s3Region: data.s3Region, + } + destinationConfig = { iceberg: icebergConfig } } const batchConfig: any = {} @@ -198,7 +280,7 @@ export const DestinationPanel = ({ pipelineId: existingDestination.pipelineId, projectRef, destinationName: data.name, - destinationConfig: { bigQuery: bigQueryConfig }, + destinationConfig, pipelineConfig: { publicationName: data.publicationName, ...(hasBatchFields ? { batch: batchConfig } : {}), @@ -226,15 +308,43 @@ export const DestinationPanel = ({ startPipeline({ projectRef, pipelineId: existingDestination.pipelineId }) onClose() } else { - const bigQueryConfig: any = { - projectId: data.projectId, - datasetId: data.datasetId, - serviceAccountKey: data.serviceAccountKey, + let destinationConfig: any = {} + + if (data.type === 'BigQuery') { + const bigQueryConfig: any = { + projectId: data.projectId, + datasetId: data.datasetId, + serviceAccountKey: data.serviceAccountKey, + } + if (!!data.maxStalenessMins) { + bigQueryConfig.maxStalenessMins = data.maxStalenessMins + } + destinationConfig = { bigQuery: bigQueryConfig } + } else if (data.type === 'Analytics Bucket') { + let s3Keys = { accessKey: data.s3AccessKeyId, secretKey: data.s3SecretAccessKey } + + if (data.s3AccessKeyId === CREATE_NEW_KEY) { + const newKeys = await createS3AccessKey({ + projectRef, + description: `Autogenerated key for replication to ${snakeCase(warehouseName)}`, + }) + s3Keys = { accessKey: newKeys.access_key, secretKey: newKeys.secret_key } + } + + // Resolve namespace (create if needed) + const finalNamespace = await resolveNamespace(data) + + const icebergConfig: any = { + projectRef: projectRef, + warehouseName: data.warehouseName, + namespace: finalNamespace, + catalogToken: data.catalogToken, + s3AccessKeyId: s3Keys.accessKey, + s3SecretAccessKey: s3Keys.secretKey, + s3Region: data.s3Region, + } + destinationConfig = { iceberg: icebergConfig } } - if (!!data.maxStalenessMins) { - bigQueryConfig.maxStalenessMins = data.maxStalenessMins - } - const batchConfig: any = {} if (!!data.maxFillMs) batchConfig.maxFillMs = data.maxFillMs const hasBatchFields = Object.keys(batchConfig).length > 0 @@ -242,7 +352,7 @@ export const DestinationPanel = ({ const { pipeline_id: pipelineId } = await createDestinationPipeline({ projectRef, destinationName: data.name, - destinationConfig: { bigQuery: bigQueryConfig }, + destinationConfig, sourceId, pipelineConfig: { publicationName: data.publicationName, @@ -288,15 +398,16 @@ export const DestinationPanel = ({ } useEffect(() => { - if (editMode && destinationData && pipelineData) { + if (editMode && destinationData && pipelineData && !isFormInteracting) { form.reset(defaultValues) } - }, [destinationData, pipelineData, editMode, defaultValues, form]) + }, [destinationData, pipelineData, editMode, defaultValues, form, isFormInteracting]) // Ensure the form always reflects the freshest data whenever the panel opens useEffect(() => { if (visible) { form.reset(defaultValues) + setIsFormInteracting(false) } }, [visible, defaultValues, form]) @@ -399,8 +510,8 @@ export const DestinationPanel = ({ -
-

Where to send that data

+
+

Where to send that data

- + { + setIsFormInteracting(true) + field.onChange(value) + }} + > {field.value} BigQuery + + Analytics Bucket + @@ -424,122 +545,25 @@ export const DestinationPanel = ({ )} /> - ( - - - - - - )} - /> - - ( - - - - - - )} - /> - - ( - - - - - - )} - /> + {selectedType === 'BigQuery' ? ( + + ) : selectedType === 'Analytics Bucket' ? ( + + ) : null}
- - - - Advanced Settings - - - ( - - - { - const val = e.target.value - field.onChange(val === '' ? undefined : Number(val)) - }} - placeholder="Leave empty for default" - /> - - - )} - /> - ( - - - { - const val = e.target.value - field.onChange(val === '' ? undefined : Number(val)) - }} - placeholder="Leave empty for default" - /> - - - )} - /> - - - +
+ + ) : isErrorBuckets ? ( + + ) : ( + + { + setIsFormInteracting(true) + field.onChange(value) + // [Joshen] Ideally should select the first namespace of the selected bucket + form.setValue('namespace', '') + }} + > + {field.value || 'Select a bucket'} + + + {analyticsBuckets.length === 0 ? ( + + No buckets available + + ) : ( + analyticsBuckets.map((bucket) => ( + + {bucket.name} + + )) + )} + + + + + )} + + )} + /> + + ( + + {isLoadingNamespaces && canSelectNamespace ? ( + + ) : isErrorNamespaces ? ( + + ) : ( + + { + setIsFormInteracting(true) + field.onChange(value) + }} + disabled={!canSelectNamespace} + > + + {!canSelectNamespace + ? 'Select a warehouse first' + : field.value === CREATE_NEW_NAMESPACE + ? 'Create a new namespace' + : field.value || 'Select a namespace'} + + + + {namespaces.length === 0 ? ( + + No namespaces available + + ) : ( + namespaces.map((namespace) => ( + + {namespace} + + )) + )} + {namespaces.length > 0 && } + + Create a new namespace + + + + + + )} + + )} + /> + + {namespace === CREATE_NEW_NAMESPACE && ( + ( + + + + + + )} + /> + )} + + + +

Credentials

+ + ( + + Automatically retrieved from your project's{' '} + + service role key + + + } + > + +
+ ) : null + } + /> + + )} + /> + + ( + + {isLoadingKeys ? ( + + ) : isErrorKeys ? ( + + ) : ( + + + + {field.value === CREATE_NEW_KEY + ? 'Create a new key' + : (field.value ?? '').length === 0 + ? 'Select an access key ID' + : field.value} + + + + {s3Keys.map((key) => ( + + {key.access_key} +

{key.description}

+ + ))} + + + Create a new key + + + + + + )} + + )} + /> + + {isSuccessKeys && keyNoLongerExists && ( +
+ +

+ Please select another key or create a new set, as this destination will not work + otherwise. S3 access keys can be managed in your{' '} + + storage settings + +

+ +
+ )} + + {s3AccessKeyId === CREATE_NEW_KEY ? ( +
+ +

+ S3 access keys can be managed in your{' '} + + storage settings + + . +

+ +
+ ) : ( + ( + + + + +