Ingest v2: Zendesk source connector (#513)

Author: Paul-Cornell

ingestion/source-connectors/zendesk.mdx

@@ -0,0 +1,27 @@
+---
+title: Zendesk
+---
+
+import NewDocument from '/snippets/general-shared-text/new-document.mdx';
+
+<NewDocument />
+
+import SharedContentZendesk from '/snippets/sc-shared-text/zendesk-cli-api.mdx';
+
+<SharedContentZendesk/>
+
+Now call the Unstructured Ingest CLI or the Unstructured Ingest Python library. The destination connector can be any of the ones supported. This example uses the local destination connector.
+
+This example sends data to Unstructured for processing by default. To process data locally instead, see the instructions at the end of this page.
+
+import ZendeskSh from '/snippets/source_connectors/zendesk.sh.mdx';
+import ZendeskPyV2 from '/snippets/source_connectors/zendesk.v2.py.mdx';
+
+<CodeGroup>
+  <ZendeskSh />
+  <ZendeskPyV2 />
+</CodeGroup>
+
+import SharedPartitionByAPIOSS from '/snippets/ingest-configuration-shared/partition-by-api-oss.mdx';
+
+<SharedPartitionByAPIOSS/>

mint.json

@@ -443,7 +443,8 @@
               "ingestion/source-connectors/slack",
               "ingestion/source-connectors/snowflake",
               "ingestion/source-connectors/sqlite",
-              "ingestion/source-connectors/wikipedia"
+              "ingestion/source-connectors/wikipedia",
+              "ingestion/source-connectors/zendesk"
             ]
           },
           {

snippets/general-shared-text/zendesk-cli-api.mdx

@@ -0,0 +1,15 @@
+The Zendesk connector dependencies:
+
+```bash CLI, Python
+pip install "unstructured-ingest[zendesk]"
+```
+
+import AdditionalIngestDependencies from '/snippets/general-shared-text/ingest-dependencies.mdx';
+
+<AdditionalIngestDependencies />
+
+The following environment variables:
+
+- `ZENDESK_SUBDOMAIN` - The subdomain for your Zendesk account, represented by `--subdomain` (CLI) or `subdomain` (Python).
+- `ZENDESK_EMAIL` - The login email address for your Zendesk account, represented by `--email` (CLI) or `email` (Python).
+- `ZENDESK_API_TOKEN` - The API token (not OAuth token) for your login email address, represented by `--api-token` (CLI) or `api_token` (Python).

snippets/general-shared-text/zendesk.mdx

@@ -0,0 +1,4 @@
+- A [Zendesk account](https://www.zendesk.com/register/).
+- Your [Zendesk subdomain](https://support.zendesk.com/hc/en-us/articles/4409381383578-Where-can-I-find-my-Zendesk-subdomain).
+- The login email address for your Zendesk account.
+- An [API token](https://support.zendesk.com/hc/en-us/articles/4408889192858-Managing-access-to-the-Zendesk-API#topic_tcb_fk1_2yb) (not an OAuth token) for your login email address.

snippets/sc-shared-text/zendesk-cli-api.mdx

@@ -0,0 +1,9 @@
+Connect Zendesk to your preprocessing pipeline, and use the Unstructured CLI or Python to batch process all your documents and store structured outputs locally on your filesystem.
+
+The requirements are as follows. 
+
+import SharedZendesk from '/snippets/general-shared-text/zendesk.mdx';
+import SharedZendeskCLIAPI from '/snippets/general-shared-text/zendesk-cli-api.mdx';
+
+<SharedZendesk />
+<SharedZendeskCLIAPI />

snippets/source_connectors/zendesk.sh.mdx

@@ -0,0 +1,19 @@
+```bash CLI
+#!/usr/bin/env bash
+
+# Chunking and embedding are optional.
+
+unstructured-ingest \
+  zendesk \
+    --subdomain $ZENDESK_SUBDOMAIN \
+    --email $ZENDESK_EMAIL \
+    --api-token $ZENDESK_API_TOKEN \
+    --batch-size 2 \
+    --item-type tickets \
+    --output-dir $LOCAL_FILE_OUTPUT_DIR \
+    --partition-by-api \
+    --api-key $UNSTRUCTURED_API_KEY \
+    --partition-endpoint $UNSTRUCTURED_API_URL \
+    --chunking-strategy by_title \
+    --embedding-provider huggingface
+```

snippets/source_connectors/zendesk.v2.py.mdx

@@ -0,0 +1,50 @@
+```python Python Ingest v2
+import os
+
+from unstructured_ingest.v2.pipeline.pipeline import Pipeline
+from unstructured_ingest.v2.interfaces import ProcessorConfig
+
+from unstructured_ingest.v2.processes.connectors.zendesk import (
+    ZendeskIndexerConfig,
+    ZendeskDownloaderConfig,
+    ZendeskConnectionConfig,
+    ZendeskAccessConfig
+)
+
+from unstructured_ingest.v2.processes.partitioner import PartitionerConfig
+from unstructured_ingest.v2.processes.chunker import ChunkerConfig
+from unstructured_ingest.v2.processes.embedder import EmbedderConfig
+from unstructured_ingest.v2.processes.connectors.local import LocalUploaderConfig
+
+# Chunking and embedding are optional.
+
+if __name__ == "__main__":
+    Pipeline.from_configs(
+        context=ProcessorConfig(),
+        indexer_config=ZendeskIndexerConfig(
+            batch_size=2,
+            item_type=["tickets"]
+        ),
+        downloader_config=ZendeskDownloaderConfig(download_dir=os.getenv("LOCAL_FILE_DOWNLOAD_DIR")),
+        source_connection_config=ZendeskConnectionConfig(
+            access_config=ZendeskAccessConfig(
+                api_token=os.getenv("ZENDESK_API_TOKEN")
+            ),
+            subdomain=os.getenv("ZENDESK_SUBDOMAIN"),
+            email=os.getenv("ZENDESK_EMAIL")
+        ),
+        partitioner_config=PartitionerConfig(
+            partition_by_api=True,
+            api_key=os.getenv("UNSTRUCTURED_API_KEY"),
+            partition_endpoint=os.getenv("UNSTRUCTURED_API_URL"),
+            additional_partition_args={
+                "split_pdf_page": True,
+                "split_pdf_allow_failed": True,
+                "split_pdf_concurrency_level": 15
+            }
+        ),
+        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        uploader_config=LocalUploaderConfig(output_dir=os.getenv("LOCAL_FILE_OUTPUT_DIR"))
+    ).run()
+```