Platform: Delta Tables in Databricks destination connector (#457)

Paul-Cornell · web-flow · commit d75c7bcb4042 · 2025-01-30T10:20:30.000-08:00
diff --git a/mint.json b/mint.json
@@ -506,6 +506,7 @@
               "platform/destinations/couchbase",
               "platform/destinations/databricks-volumes",
               "platform/destinations/delta-table",
+              "platform/destinations/databricks-delta-table",
               "platform/destinations/elasticsearch",
               "platform/destinations/google-cloud",
               "platform/destinations/kafka",
@@ -557,6 +558,7 @@
                   "platform/api/destinations/couchbase",
                   "platform/api/destinations/databricks-volumes",
                   "platform/api/destinations/delta-table",
+                  "platform/api/destinations/databricks-delta-table",
                   "platform/api/destinations/elasticsearch",
                   "platform/api/destinations/google-cloud",
                   "platform/api/destinations/kafka",
diff --git a/platform/api/destinations/databricks-delta-table.mdx b/platform/api/destinations/databricks-delta-table.mdx
@@ -0,0 +1,40 @@
+---
+title: Delta Tables in Databricks
+---
+
+<Note>
+    This article covers connecting Unstructured to Delta Tables in Databricks.
+    
+    For information about connecting Unstructured to Delta Tables in Amazon S3 instead, see 
+    [Delta Tables in Amazon S3](/platform/api/destinations/delta-table).
+
+    For information about connecting Unstructured to Databricks Volumes instead, see 
+    [Databricks Volumes](/platform/api/destinations/databricks-volumes).
+</Note>
+
+Send processed data from Unstructured to a Delta Table in Databricks.
+
+The requirements are as follows.
+
+import DeltaTablesInDatabricksPrerequisites from '/snippets/general-shared-text/databricks-delta-table.mdx';
+
+<DeltaTablesInDatabricksPrerequisites />
+
+To create or change a Delta Tables in Databricks destination connector, see the following examples.
+
+import DeltaTablesInDatabricksAPIRESTCreate from '/snippets/destination_connectors/databricks_delta_table_rest_create.mdx';
+import DeltaTablesInDatabricksAPIRESTChange from '/snippets/destination_connectors/databricks_delta_table_rest_change.mdx';
+
+<CodeGroup>
+    <DeltaTablesInDatabricksAPIRESTCreate />
+    <DeltaTablesInDatabricksAPIRESTChange />
+</CodeGroup>
+
+Replace the preceding placeholders as follows:
+
+import DeltaTablesInDatabricksAPIPlaceholders from '/snippets/general-shared-text/databricks-delta-table-api-placeholders.mdx';
+
+<DeltaTablesInDatabricksAPIPlaceholders />
+
+To change a connector, replace `<connector-id>` with the source connector's unique ID. 
+To get this ID, see [List source connectors](/platform/api/overview#list-source-connectors).
diff --git a/platform/api/destinations/databricks-volumes.mdx b/platform/api/destinations/databricks-volumes.mdx
@@ -2,6 +2,13 @@
 title: Databricks Volumes
 ---
 
+<Note>
+    This article covers connecting Unstructured to Databricks Volumes.
+    
+    For information about connecting Unstructured to Delta Tables in Databricks instead, see 
+    [Delta Tables in Databricks](/platform/api/destinations/databricks-delta-table).
+</Note>
+
 Send processed data from Unstructured to Databricks Volumes.
 
 The requirements are as follows.
diff --git a/platform/api/destinations/delta-table.mdx b/platform/api/destinations/delta-table.mdx
@@ -1,7 +1,13 @@
 ---
-title: Delta Table
+title: Delta Tables in Amazon S3
 ---
 
+<Note>
+    This article covers connecting Unstructured to Delta Tables in Amazon S3. For information about 
+    connecting Unstructured to Delta Tables in Databricks instead, see 
+    [Delta Tables in Databricks](/platform/api/destinations/databricks-delta-table).
+</Note>
+
 Send processed data from Unstructured to a Delta Table, stored in Amazon S3.
 
 The requirements are as follows.
@@ -10,7 +16,7 @@ import DeltaTablePrerequisites from '/snippets/general-shared-text/delta-table.m
 
 <DeltaTablePrerequisites />
 
-To create or change a Delta Table destination connector, see the following examples.
+To create or change a Delta Tables in Amazon S3 destination connector, see the following examples.
 
 import DeltaTableAPIRESTCreate from '/snippets/destination_connectors/delta_table_rest_create.mdx';
 import DeltaTableAPIRESTChange from '/snippets/destination_connectors/delta_table_rest_change.mdx';
diff --git a/platform/api/destinations/overview.mdx b/platform/api/destinations/overview.mdx
@@ -17,7 +17,8 @@ For the list of specific settings, see:
 - [Azure AI Search](/platform/api/destinations/azure-ai-search) (`destination_type=azure_ai_search`)
 - [Couchbase](/platform/api/destinations/couchbase) (`destination_type=couchbase`)
 - [Databricks Volumes](/platform/api/destinations/databricks-volumes) (`destination_type=databricks_volumes`)
-- [Delta Table](/platform/api/destinations/delta-table) (`destination_type=delta_table`)
+- [Delta Tables in Amazon S3](/platform/api/destinations/delta-table) (`destination_type=delta_table`)
+- [Delta Tables in Databricks](/platform/api/destinations/databricks-delta-table) (`destination_type=databricks_volume_delta_tables`)
 - [Elasticsearch](/platform/api/destinations/elasticsearch) (`destination_type=elasticsearch`)
 - [Google Cloud Storage](/platform/api/destinations/google-cloud) (`destination_type=gcs`)
 - [Kafka](/platform/api/destinations/kafka) (`destination_type=kafka-cloud`)
diff --git a/platform/connectors.mdx b/platform/connectors.mdx
@@ -39,7 +39,8 @@ If your source is not listed here, you might still be able to connect Unstructur
 - [Azure AI Search](/platform/destinations/azure-ai-search)
 - [Couchbase](/platform/destinations/couchbase)
 - [Databricks Volumes](/platform/destinations/databricks-volumes)
-- [Delta Table](/platform/destinations/delta-table)
+- [Delta Tables in Amazon S3](/platform/destinations/delta-table)
+- [Delta Tables in Databricks](/platform/destinations/databricks-delta-table)
 - [Elasticsearch](/platform/destinations/elasticsearch)
 - [Google Cloud Storage](/platform/destinations/google-cloud)
 - [Kafka](/platform/destinations/kafka)
diff --git a/platform/destinations/databricks-delta-table.mdx b/platform/destinations/databricks-delta-table.mdx
@@ -0,0 +1,36 @@
+---
+title: Delta Tables in Databricks
+---
+
+<Note>
+    This article covers connecting Unstructured to Delta Tables in Databricks.
+    
+    For information about connecting Unstructured to Delta Tables in Amazon S3 instead, see 
+    [Delta Tables in Amazon S3](/platform/destinations/delta-table).
+
+    For information about connecting Unstructured to Databricks Volumes instead, see 
+    [Databricks Volumes](/platform/destinations/databricks-volumes).
+</Note>
+
+Send processed data from Unstructured to a Delta Table in Databricks.
+
+The requirements are as follows.
+
+import DeltaTablesInDatabricksPrerequisites from '/snippets/general-shared-text/databricks-delta-table.mdx';
+
+<DeltaTablesInDatabricksPrerequisites />
+
+To create the destination connector:
+
+1. On the sidebar, click **Connectors**.
+2. Click **Destinations**.
+3. Cick **New** or **Create Connector**.
+4. Give the connector some unique **Name**.
+5. In the **Provider** area, click **Delta Tables in Databricks**.
+6. Click **Continue**. 
+7. Follow the on-screen instructions to fill in the fields as described later on this page.
+8. Click **Save and Test**.
+
+import DeltaTablesInDatabricksFields from '/snippets/general-shared-text/databricks-delta-table-platform.mdx';
+
+<DeltaTablesInDatabricksFields />
diff --git a/platform/destinations/databricks-volumes.mdx b/platform/destinations/databricks-volumes.mdx
@@ -2,6 +2,13 @@
 title: Databricks Volumes
 ---
 
+<Note>
+    This article covers connecting Unstructured to Databricks Volumes.
+    
+    For information about connecting Unstructured to Delta Tables in Databricks instead, see 
+    [Delta Tables in Databricks](/platform/destinations/databricks-delta-table).
+</Note>
+
 Send processed data from Unstructured to Databricks Volumes.
 
 The requirements are as follows.
diff --git a/platform/destinations/delta-table.mdx b/platform/destinations/delta-table.mdx
@@ -1,7 +1,13 @@
 ---
-title: Delta Table
+title: Delta Tables in Amazon S3
 ---
 
+<Note>
+    This article covers connecting Unstructured to Delta Tables in Amazon S3. For information about 
+    connecting Unstructured to Delta Tables in Databricks instead, see 
+    [Delta Tables in Databricks](/platform/destinations/databricks-delta-table).
+</Note>
+
 Send processed data from Unstructured to a Delta Table, stored in Amazon S3.
 
 The requirements are as follows.
diff --git a/platform/destinations/overview.mdx b/platform/destinations/overview.mdx
@@ -21,7 +21,8 @@ To create a destination connector:
    - [Azure AI Search](/platform/destinations/azure-ai-search)
    - [Couchbase](/platform/destinations/couchbase)
    - [Databricks Volumes](/platform/destinations/databricks-volumes)
-   - [Delta Table](/platform/destinations/delta-table)
+   - [Delta Tables in Amazon S3](/platform/destinations/delta-table)
+   - [Delta Tables in Databricks](/platform/destinations/databricks-delta-table)
    - [Elasticsearch](/platform/destinations/elasticsearch)
    - [Google Cloud Storage](/platform/destinations/google-cloud)
    - [Kafka](/platform/destinations/kafka)
diff --git a/snippets/destination_connectors/databricks_delta_table_rest_change.mdx b/snippets/destination_connectors/databricks_delta_table_rest_change.mdx
@@ -0,0 +1,23 @@
+```bash REST (Change)
+curl --request 'PUT' --location \
+"$UNSTRUCTURED_API_URL/destinations/<connector-id>" \
+--header 'accept: application/json' \
+--header "unstructured-api-key: $UNSTRUCTURED_API_KEY" \
+--header 'content-type: application/json' \
+--data \
+'{
+    "config": {
+        "server_hostname": "<server-hostname>",
+        "http_path": "<http-path>",
+        "token": "<token>",
+        "client_id": "<client-id>",
+        "client_secret": "<client-secret>",
+        "volume": "<volume>",
+        "catalog": "<catalog>",
+        "volume_path": "<volume-path>",
+        "schema": "<schema>",
+        "database": "<database>",
+        "table_name": "<table-name>"
+    }
+}'
+```
diff --git a/snippets/destination_connectors/databricks_delta_table_rest_create.mdx b/snippets/destination_connectors/databricks_delta_table_rest_create.mdx
@@ -0,0 +1,25 @@
+```bash REST (Create)
+curl --request 'POST' --location \
+"$UNSTRUCTURED_API_URL/destinations" \
+--header 'accept: application/json' \
+--header "unstructured-api-key: $UNSTRUCTURED_API_KEY" \
+--header 'content-type: application/json' \
+--data \
+'{
+    "name": "<name>",
+    "type": "databricks_volume_delta_tables",
+    "config": {
+        "server_hostname": "<server-hostname>",
+        "http_path": "<http-path>",
+        "token": "<token>",
+        "client_id": "<client-id>",
+        "client_secret": "<client-secret>",
+        "volume": "<volume>",
+        "catalog": "<catalog>",
+        "volume_path": "<volume-path>",
+        "schema": "<schema>",
+        "database": "<database>",
+        "table_name": "<table-name>"
+    }
+}'
+```
diff --git a/snippets/general-shared-text/databricks-delta-table-api-placeholders.mdx b/snippets/general-shared-text/databricks-delta-table-api-placeholders.mdx
@@ -0,0 +1,14 @@
+- `<name>` (_required_) - A unique name for this connector.
+- `<server-hostname>` (_required_): The target Databricks cluster's or SQL warehouse's **Server Hostname** value.
+- `<http-path>` (_required_): The cluster's or SQL warehouse's **HTTP Path** value.
+- `<token>` (_required_ for PAT authentication): For Databricks personal access token (PAT) authentication, the target Databricks user's PAT value.
+- `<client-id>` and `<client-secret>` (_required_ for OAuth authentication): For Databricks OAuth machine-to-machine (M2M) authentication, the Databricks managed service principal's **UUID** (client ID or Application ID) and OAuth **Secret** (client secret) values. 
+- `<catalog>` (_required_): The name of the catalog in Unity Catalog for the target volume and table in the Databricks workspace.
+- `<database>`: The name of the database in Unity Catalog for the target volume and table. The default is `default` if not otherwise specified.
+- `<table-name>` (_required_): The name of the target table in Unity Catalog.
+- `<schema>`: The name of the schema in Unity Catalog for the target volume and table. The default is `default` if not otherwise specified.
+- `<volume>` (_required_): The name of the target volume in Unity Catalog.
+- `<volume-path>`: Any target folder path inside of the volume to use instead of the volume's root. If not otherwise specified, processing occurs at the volume's root.
+
+
+
diff --git a/snippets/general-shared-text/databricks-delta-table-platform.mdx b/snippets/general-shared-text/databricks-delta-table-platform.mdx
@@ -0,0 +1,13 @@
+Fill in the following fields:
+
+- **Name** (_required_): A unique name for this connector.
+- **Server Hostname** (_required_): The target Databricks cluster's or SQL warehouse's **Server Hostname** value.
+- **HTTP Path** (_required_): The cluster's or SQL warehouse's **HTTP Path** value.
+- **Token** (_required_ for PAT authentication): For Databricks personal access token (PAT) authentication, the target Databricks user's PAT value.
+- **UUID** and **OAuth Secret** (_required_ for OAuth authentication): For Databricks OAuth machine-to-machine (M2M) authentication, the Databricks managed service principal's **UUID** (client ID or Application ID) and OAuth **Secret** (client secret) values. 
+- **Catalog** (_required_): The name of the catalog in Unity Catalog for the target volume and table in the Databricks workspace.
+- **Database**: The name of the database in Unity Catalog for the target volume and table. The default is `default` if not otherwise specified.
+- **Table Name** (_required_): The name of the target table in Unity Catalog.
+- **Schema**: The name of the schema in Unity Catalog for the target volume and table. The default is `default` if not otherwise specified.
+- **Volume** (_required_): The name of the target volume in Unity Catalog.
+- **Volume Path**: Any target folder path inside of the volume to use instead of the volume's root. If not otherwise specified, processing occurs at the volume's root.
