S3 source connectors: how to work with user-defined metadata (#696)

Paul-Cornell · web-flow · commit 430db53fb438 · 2025-07-30T11:43:01.000-07:00
diff --git a/api-reference/workflow/destinations/s3.mdx b/api-reference/workflow/destinations/s3.mdx
@@ -14,6 +14,26 @@ import s3Prerequisites from '/snippets/general-shared-text/s3.mdx';
 
 <s3Prerequisites />
 
+## Add an access policy to an existing bucket
+
+import S3BucketPolicy from '/snippets/general-shared-text/s3-bucket-policy.mdx';
+
+<S3BucketPolicy />
+
+## Create a bucket with AWS CloudFormation
+
+import S3BucketCloudFormation from '/snippets/general-shared-text/s3-cf-setup.mdx';
+
+<S3BucketCloudFormation />
+
+## Create a bucket with the AWS CLI
+
+import S3BucketCLI from '/snippets/general-shared-text/s3-cli-setup.mdx';
+
+<S3BucketCLI />
+
+## Create the destination connector
+
 To create an S3 destination connector, see the following examples.
 
 import s3SDK from '/snippets/destination_connectors/s3_sdk.mdx';
diff --git a/api-reference/workflow/sources/s3.mdx b/api-reference/workflow/sources/s3.mdx
@@ -14,6 +14,32 @@ import S3Prerequisites from '/snippets/general-shared-text/s3.mdx';
 
 <S3Prerequisites />
 
+## Add an access policy to an existing bucket
+
+import S3BucketPolicy from '/snippets/general-shared-text/s3-bucket-policy.mdx';
+
+<S3BucketPolicy />
+
+## Create a bucket with AWS CloudFormation
+
+import S3BucketCloudFormation from '/snippets/general-shared-text/s3-cf-setup.mdx';
+
+<S3BucketCloudFormation />
+
+## Create a bucket with the AWS CLI
+
+import S3BucketCLI from '/snippets/general-shared-text/s3-cli-setup.mdx';
+
+<S3BucketCLI />
+
+## Work with user-defined metadata
+
+import S3Metadata from '/snippets/general-shared-text/s3-metadata.mdx';
+
+<S3Metadata />
+
+## Create the source connector
+
 To create an S3 source connector, see the following examples.
 
 import S3SDK from '/snippets/source_connectors/s3_sdk.mdx';
diff --git a/open-source/ingestion/destination-connectors/s3.mdx b/open-source/ingestion/destination-connectors/s3.mdx
@@ -10,22 +10,6 @@ import SharedS3 from '/snippets/dc-shared-text/s3-cli-api.mdx';
 
 <SharedS3 />
 
-Now call the Unstructured Ingest CLI or the Unstructured Ingest Python library. The source connector can be any of the ones supported. This example uses the local source connector.
-
-This example sends files to Unstructured for processing by default. To process files locally instead, see the instructions at the end of this page.
-
-import S3APISh from '/snippets/destination_connectors/s3.sh.mdx';
-import S3APIPyV2 from '/snippets/destination_connectors/s3.v2.py.mdx';
-
-<CodeGroup>
-  <S3APISh />
-  <S3APIPyV2 />
-</CodeGroup>
-
-import SharedPartitionByAPIOSS from '/snippets/ingest-configuration-shared/partition-by-api-oss.mdx';
-
-<SharedPartitionByAPIOSS/>
-
 ## Add an access policy to an existing bucket
 
 import S3BucketPolicy from '/snippets/general-shared-text/s3-bucket-policy.mdx';
@@ -43,3 +27,21 @@ import S3BucketCLI from '/snippets/general-shared-text/s3-cli-setup.mdx';
 import S3BucketCloudFormation from '/snippets/general-shared-text/s3-cf-setup.mdx';
 
 <S3BucketCloudFormation />
+
+## Create a pipeline that uses S3 as the destination
+
+Now call the Unstructured Ingest CLI or the Unstructured Ingest Python library. The source connector can be any of the ones supported. This example uses the local source connector.
+
+This example sends files to Unstructured for processing by default. To process files locally instead, see the instructions at the end of this page.
+
+import S3APISh from '/snippets/destination_connectors/s3.sh.mdx';
+import S3APIPyV2 from '/snippets/destination_connectors/s3.v2.py.mdx';
+
+<CodeGroup>
+  <S3APISh />
+  <S3APIPyV2 />
+</CodeGroup>
+
+import SharedPartitionByAPIOSS from '/snippets/ingest-configuration-shared/partition-by-api-oss.mdx';
+
+<SharedPartitionByAPIOSS/>
diff --git a/open-source/ingestion/source-connectors/s3.mdx b/open-source/ingestion/source-connectors/s3.mdx
@@ -10,6 +10,32 @@ import SharedContentS3 from '/snippets/sc-shared-text/s3-cli-api.mdx';
 
 <SharedContentS3/>
 
+## Add an access policy to an existing bucket
+
+import S3BucketPolicy from '/snippets/general-shared-text/s3-bucket-policy.mdx';
+
+<S3BucketPolicy />
+
+## Create a bucket with the AWS CLI
+
+import S3BucketCLI from '/snippets/general-shared-text/s3-cli-setup.mdx';
+
+<S3BucketCLI />
+
+## Create a bucket with AWS CloudFormation
+
+import S3BucketCloudFormation from '/snippets/general-shared-text/s3-cf-setup.mdx';
+
+<S3BucketCloudFormation />
+
+## Work with user-defined metadata
+
+import S3Metadata from '/snippets/general-shared-text/s3-metadata.mdx';
+
+<S3Metadata />
+
+## Create a pipeline that uses S3 as the source 
+
 Now call the Unstructured Ingest CLI or the Unstructured Ingest Python library. The destination connector can be any of the ones supported.
 
 <iframe
@@ -38,20 +64,3 @@ import SharedPartitionByAPIOSS from '/snippets/ingest-configuration-shared/parti
 
 <SharedPartitionByAPIOSS/>
 
-## Add an access policy to an existing bucket
-
-import S3BucketPolicy from '/snippets/general-shared-text/s3-bucket-policy.mdx';
-
-<S3BucketPolicy />
-
-## Create a bucket with the AWS CLI
-
-import S3BucketCLI from '/snippets/general-shared-text/s3-cli-setup.mdx';
-
-<S3BucketCLI />
-
-## Create a bucket with AWS CloudFormation
-
-import S3BucketCloudFormation from '/snippets/general-shared-text/s3-cf-setup.mdx';
-
-<S3BucketCloudFormation />
diff --git a/snippets/general-shared-text/s3-metadata.mdx b/snippets/general-shared-text/s3-metadata.mdx
@@ -0,0 +1,33 @@
+User-defined metadata in S3 is metadata that you can choose to set at the time that you upload a file to an S3 bucket. 
+User-defined metadata is specified in S3 as a set of name-value pairs. Each name-value pair begins with `x-amz-meta-` and is 
+followed by a unique name.
+
+For more information about how to add or replace user-defined metadata for a file in S3, see the following:
+
+- [Working with object metadata](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html)
+- [User-defined metadata](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html#UserMetadata)
+- [Editing object metadata in the Amazon S3 console](https://docs.aws.amazon.com/AmazonS3/latest/userguide/add-object-metadata.html)
+
+Unstructured outputs any user-defined metadata that it finds for a file into the `metadata.data_source.record_locator.metadata` field of 
+the document elements' output for the corresponding file. For example, if Unstructured processes a file with the user-defined metadata 
+`x-amz-meta-mymetadata` name set to the value `myvalue`, Unstructured outputs the following into the `metadata.data_source.record_locator.metadata` field of 
+the document elements' output for the corresponding file:
+
+```json
+[
+    {
+        "type": "...",
+        "element_id": "...",
+        "text": "...",
+        "metadata": {
+            "data_source": {
+                "record_locator": {
+                    "metadata": {
+                        "mymetadata": "myvalue"
+                    }
+                }
+            }
+        }
+    }
+]
+```
diff --git a/ui/destinations/s3.mdx b/ui/destinations/s3.mdx
@@ -14,21 +14,6 @@ import S3Prerequisites from '/snippets/general-shared-text/s3.mdx';
 
 <S3Prerequisites />
 
-To create the destination connector:
-
-1. On the sidebar, click **Connectors**.
-2. Click **Destinations**.
-3. Cick **New** or **Create Connector**.
-4. Give the connector some unique **Name**.
-5. In the **Provider** area, click **Amazon S3**.
-6. Click **Continue**. 
-7. Follow the on-screen instructions to fill in the fields as described later on this page.
-8. Click **Save and Test**.
-
-import S3Fields from '/snippets/general-shared-text/s3-platform.mdx';
-
-<S3Fields />
-
 ## Add an access policy to an existing bucket
 
 import S3BucketPolicy from '/snippets/general-shared-text/s3-bucket-policy.mdx';
@@ -47,3 +32,22 @@ import S3BucketCLI from '/snippets/general-shared-text/s3-cli-setup.mdx';
 
 <S3BucketCLI />
 
+## Create the destination connector
+
+To create the destination connector:
+
+1. On the sidebar, click **Connectors**.
+2. Click **Destinations**.
+3. Click **New** or **Create Connector**.
+4. Give the connector some unique **Name**.
+5. In the **Provider** area, click **Amazon S3**.
+6. Click **Continue**. 
+7. Follow the on-screen instructions to fill in the fields as described later on this page.
+8. Click **Save and Test**.
+
+import S3Fields from '/snippets/general-shared-text/s3-platform.mdx';
+
+<S3Fields />
+
+
+
diff --git a/ui/sources/s3.mdx b/ui/sources/s3.mdx
@@ -14,21 +14,6 @@ import S3Prerequisites from '/snippets/general-shared-text/s3.mdx';
 
 <S3Prerequisites />
 
-To create the source connector:
-
-1. On the sidebar, click **Connectors**.
-2. Click **Sources**.
-3. Cick **New** or **Create Connector**.
-4. Give the connector some unique **Name**.
-5. In the **Provider** area, click **Amazon S3**.
-6. Click **Continue**. 
-7. Follow the on-screen instructions to fill in the fields as described later on this page.
-8. Click **Save and Test**.
-
-import S3Fields from '/snippets/general-shared-text/s3-platform.mdx';
-
-<S3Fields />
-
 ## Add an access policy to an existing bucket
 
 import S3BucketPolicy from '/snippets/general-shared-text/s3-bucket-policy.mdx';
@@ -45,4 +30,27 @@ import S3BucketCloudFormation from '/snippets/general-shared-text/s3-cf-setup.md
 
 import S3BucketCLI from '/snippets/general-shared-text/s3-cli-setup.mdx';
 
-<S3BucketCLI />
+<S3BucketCLI />
+
+## Work with user-defined metadata
+
+import S3Metadata from '/snippets/general-shared-text/s3-metadata.mdx';
+
+<S3Metadata />
+
+## Create the source connector
+
+To create the source connector:
+
+1. On the sidebar, click **Connectors**.
+2. Click **Sources**.
+3. Click **New** or **Create Connector**.
+4. Give the connector some unique **Name**.
+5. In the **Provider** area, click **Amazon S3**.
+6. Click **Continue**. 
+7. Follow the on-screen instructions to fill in the fields as described later on this page.
+8. Click **Save and Test**.
+
+import S3Fields from '/snippets/general-shared-text/s3-platform.mdx';
+
+<S3Fields />
