Outlook v2 source connector (#286)

Paul-Cornell · web-flow · commit 38456203ac4e · 2024-10-22T16:20:45.000-07:00
diff --git a/api-reference/ingest/source-connectors/outlook.mdx b/api-reference/ingest/source-connectors/outlook.mdx
@@ -2,22 +2,24 @@
 title: Outlook
 ---
 
-import SharedContentOutlook from '/snippets/sc-shared-text/outlook.mdx';
+import NewDocument from '/snippets/general-shared-text/new-document.mdx';
+
+<NewDocument />
+
+import SharedContentOutlook from '/snippets/sc-shared-text/outlook-cli-api.mdx';
+import SharedAPIKeyURL from '/snippets/general-shared-text/api-key-url.mdx';
 
 <SharedContentOutlook/>
+<SharedAPIKeyURL/>
 
-Make sure to set the `--partition-by-api` flag and pass in your API key with `--api-key`:
+Now call the Unstructured CLI or Python SDK. The destination connector can be any of the ones supported. This example uses the local destination connector:
 
-import OutlookAPISh from '/snippets/source_connectors/outlook_api.sh.mdx';
-import OutlookAPIPy from '/snippets/source_connectors/outlook_api.py.mdx';
+import OutlookAPISh from '/snippets/source_connectors/outlook.sh.mdx';
+import OutlookAPIPyV2 from '/snippets/source_connectors/outlook.v2.py.mdx';
+import OutlookAPIPyV1 from '/snippets/source_connectors/outlook.v1.py.mdx';
 
 <CodeGroup>
-
   <OutlookAPISh />
-
-  <OutlookAPIPy />
-
-</CodeGroup>
-
-Additionally, if you're using Unstructured Serverless API, your locally deployed Unstructured API, or an Unstructured API
-deployed on Azure or AWS, you also need to specify the API URL via the `--partition-endpoint` argument.
+  <OutlookAPIPyV2 />
+  <OutlookAPIPyV1 />
+</CodeGroup>
diff --git a/open-source/ingest/source-connectors/outlook.mdx b/open-source/ingest/source-connectors/outlook.mdx
@@ -2,19 +2,29 @@
 title: Outlook
 ---
 
-import SharedContentOutlook from '/snippets/sc-shared-text/outlook.mdx';
+import NewDocument from '/snippets/general-shared-text/new-document.mdx';
+
+<NewDocument />
+
+import SharedContentOutlook from '/snippets/sc-shared-text/outlook-cli-api.mdx';
 
 <SharedContentOutlook/>
 
+Now call the Unstructured CLI or Python. The destination connector can be any of the ones supported. This example uses the local destination connector:
+
+This example sends data to Unstructured API services for processing by default. To process files locally instead, see the instructions at the end of this page.
+
 import OutlookSh from '/snippets/source_connectors/outlook.sh.mdx';
-import OutlookPy from '/snippets/source_connectors/outlook.py.mdx';
+import OutlookPyV2 from '/snippets/source_connectors/outlook.v2.py.mdx';
+import OutlookPyV1 from '/snippets/source_connectors/outlook.v1.py.mdx';
 
 <CodeGroup>
-
   <OutlookSh />
+  <OutlookPyV2 />
+  <OutlookPyV1 />
+</CodeGroup>
 
-  <OutlookPy />
+import SharedPartitionByAPIOSS from '/snippets/ingest-configuration-shared/partition-by-api-oss.mdx';
 
-</CodeGroup>
+<SharedPartitionByAPIOSS/>
 
-For a full list of the options the Unstructured Ingest CLI accepts check `unstructured-ingest outlook --help`.
diff --git a/snippets/general-shared-text/outlook-cli-api.mdx b/snippets/general-shared-text/outlook-cli-api.mdx
@@ -0,0 +1,16 @@
+The Outlook connector dependencies:
+
+```bash CLI, Python
+pip install "unstructured-ingest[outlook]"
+```
+
+import AdditionalIngestDependencies from '/snippets/general-shared-text/ingest-dependencies.mdx';
+
+<AdditionalIngestDependencies />
+
+The following environment variables:
+
+- `OUTLOOK_USER_EMAIL` - The Outlook user's email address, represented by `--user-email` (CLI) or `user_mail` (Python).
+- `OUTLOOK_APP_CLIENT_ID` - The application (client) ID of the Microsoft Entra ID app registration that has access to the user's email account, represented by `--client-id` (CLI) or `client_id` (Python).
+- `OUTLOOK_APP_CLIENT_SECRET` - The client secret for the Entra ID app registration, represented by `--client-cred` (CLI) or `client_cred` (Python).
+- `OUTLOOK_APP_TENANT` - The directory (tenant) ID of the Entra ID app registration, represented by `--tenant` (CLI) or `tenant` (Python).
diff --git a/snippets/general-shared-text/outlook.mdx b/snippets/general-shared-text/outlook.mdx
@@ -0,0 +1,22 @@
+The Outlook prerequisites:
+
+<iframe
+width="560"
+height="315"
+src="https://www.youtube.com/embed/9yESRp9pzv0"
+title="YouTube video player"
+frameborder="0"
+allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
+allowfullscreen
+></iframe>
+
+- The Outlook user's email address.
+- A Microsoft Entra ID app registration in the same Azure account as the Outlook account. You will need 
+  this app registration's client (application) ID, client secret, and directory (tenant) ID. [Learn how](https://learn.microsoft.com/entra/identity-platform/quickstart-register-app).
+- The Entra ID app registration must have the following Graph API permission levels of the application (not delegated) type: 
+
+  - `Mail.Read`
+  - `Mail.ReadBasic`
+  - `User.Read.All`
+
+  [Learn how](https://learn.microsoft.com/entra/identity-platform/howto-update-permissions).
diff --git a/snippets/sc-shared-text/outlook-cli-api.mdx b/snippets/sc-shared-text/outlook-cli-api.mdx
@@ -0,0 +1,9 @@
+Connect Outlook to your preprocessing pipeline, and use the Unstructured Ingest CLI or the Unstructured Ingest Python library to batch process all your documents and store structured outputs locally on your filesystem.
+
+You will need: 
+
+import SharedOutlook from '/snippets/general-shared-text/outlook.mdx';
+import SharedOutlookCLIAPI from '/snippets/general-shared-text/outlook-cli-api.mdx';
+
+<SharedOutlook />
+<SharedOutlookCLIAPI />
diff --git a/snippets/sc-shared-text/outlook.mdx b/snippets/sc-shared-text/outlook.mdx
diff --git a/snippets/source_connectors/outlook.py.mdx b/snippets/source_connectors/outlook.py.mdx
diff --git a/snippets/source_connectors/outlook.sh.mdx b/snippets/source_connectors/outlook.sh.mdx
@@ -1,16 +1,19 @@
-```bash Shell
+```bash CLI
 #!/usr/bin/env bash
 
+# Embedding is optional.
+
 unstructured-ingest \
   outlook \
-    --client-id $MS_CLIENT_ID \
-    --client-cred $MS_CLIENT_CRED \
-    --tenant $MS_TENANT_ID \
-    --user-email $MS_USER_EMAIL \
-    --outlook-folders Inbox,"Sent Items" \
+    --user-email $OUTLOOK_USER_EMAIL \
+    --outlook-folders Inbox \
     --output-dir $LOCAL_FILE_OUTPUT_DIR \
-    --num-processes 2 \
-    --recursive \
-    --verbose \
-    --strategy hi_res
+    --client-id $OUTLOOK_APP_CLIENT_ID \
+    --client-cred $OUTLOOK_APP_CLIENT_SECRET \
+    --tenant $OUTLOOK_APP_TENANT \
+    --partition-by-api \
+    --api-key $UNSTRUCTURED_API_KEY \
+    --partition-endpoint $UNSTRUCTURED_API_URL \
+    --chunking-strategy by_title \
+    --embedding-provider huggingface
 ```
diff --git a/snippets/source_connectors/outlook.v1.py.mdx b/snippets/source_connectors/outlook.v1.py.mdx
@@ -1,4 +1,4 @@
-```python Python
+```python Python Ingest v1
 import os
 
 from unstructured_ingest.connector.outlook import OutlookAccessConfig, SimpleOutlookConfig
@@ -16,16 +16,17 @@ if __name__ == "__main__":
         partition_config=PartitionConfig(
             partition_by_api=True,
             api_key=os.getenv("UNSTRUCTURED_API_KEY"),
+            partition_endpoint=os.getenv("UNSTRUCTURED_API_URL")
             strategy="hi_res",
         ),
         connector_config=SimpleOutlookConfig(
             access_config=OutlookAccessConfig(
-                client_credential=os.getenv("MS_CLIENT_CRED"),
+                client_credential=os.getenv("OUTLOOK_APP_CLIENT_SECRET"),
             ),
-            client_id=os.getenv("MS_CLIENT_ID"),
-            tenant=os.getenv("MS_TENANT_ID"),
-            user_email=os.getenv("MS_USER_EMAIL"),
-            outlook_folders=["Inbox", "Sent Items"],
+            client_id=os.getenv("OUTLOOK_APP_CLIENT_ID"),
+            tenant=os.getenv("OUTLOOK_APP_TENANT"),
+            user_email=os.getenv("OUTLOOK_USER_EMAIL"),
+            outlook_folders=["Inbox"],
             recursive=True,
         ),
     )
diff --git a/snippets/source_connectors/outlook.v2.py.mdx b/snippets/source_connectors/outlook.v2.py.mdx
@@ -0,0 +1,44 @@
+```python Python Ingest v2
+import os
+
+from unstructured_ingest.v2.pipeline.pipeline import Pipeline
+from unstructured_ingest.v2.interfaces import ProcessorConfig
+
+from unstructured_ingest.v2.processes.connectors.outlook import (
+    OutlookIndexerConfig,
+    OutlookDownloaderConfig,
+    OutlookConnectionConfig,
+    OutlookAccessConfig
+)
+
+from unstructured_ingest.v2.processes.connectors.local import LocalUploaderConfig
+from unstructured_ingest.v2.processes.partitioner import PartitionerConfig
+from unstructured_ingest.v2.processes.chunker import ChunkerConfig
+from unstructured_ingest.v2.processes.embedder import EmbedderConfig
+
+# Embedding is optional.
+
+if __name__ == "__main__":
+    Pipeline.from_configs(
+        context=ProcessorConfig(),
+        indexer_config=OutlookIndexerConfig(
+            outlook_folders=["Inbox"],
+            recursive=False,
+            user_email=os.getenv("OUTLOOK_USER_EMAIL")
+        ),
+        downloader_config=OutlookDownloaderConfig(download_dir=os.getenv("LOCAL_FILE_DOWNLOAD_DIR")),
+        source_connection_config=OutlookConnectionConfig(
+            access_config=OutlookAccessConfig(client_cred=os.getenv("OUTLOOK_APP_CLIENT_SECRET")),
+            client_id=os.getenv("OUTLOOK_APP_CLIENT_ID"),
+            tenant=os.getenv("OUTLOOK_APP_TENANT")
+        ),
+        partitioner_config=PartitionerConfig(
+            partition_by_api=True,
+            api_key=os.getenv("UNSTRUCTURED_API_KEY"),
+            partition_endpoint=os.getenv("UNSTRUCTURED_API_URL")
+        ),
+        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        uploader_config=LocalUploaderConfig(output_dir=os.getenv("LOCAL_FILE_OUTPUT_DIR"))
+    ).run()
+```
diff --git a/snippets/source_connectors/outlook_api.sh.mdx b/snippets/source_connectors/outlook_api.sh.mdx
