Ingest: Discord source connector (#587)

Author: Paul-Cornell

docs.json

@@ -302,6 +302,7 @@
                   "ingestion/source-connectors/couchbase",
                   "ingestion/source-connectors/databricks-volumes",
                   "ingestion/source-connectors/delta-table",
+                  "ingestion/source-connectors/discord",
                   "ingestion/source-connectors/dropbox",
                   "ingestion/source-connectors/elastic-search",
                   "ingestion/source-connectors/github",

ingestion/source-connectors/discord.mdx

@@ -0,0 +1,27 @@
+---
+title: Discord
+---
+
+import NewDocument from '/snippets/general-shared-text/new-document.mdx';
+
+<NewDocument />
+
+import SharedContentDiscord from '/snippets/sc-shared-text/discord-cli-api.mdx';
+
+<SharedContentDiscord/>
+
+Now call the Unstructured CLI or Python. The destination connector can be any of the ones supported. This example uses the local destination connector.
+
+This example sends data to Unstructured for processing by default. To process data locally instead, see the instructions at the end of this page.
+
+import DiscordSh from '/snippets/source_connectors/discord.sh.mdx';
+import DiscordPyV2 from '/snippets/source_connectors/discord.v2.py.mdx';
+
+<CodeGroup>
+  <DiscordSh />
+  <DiscordPyV2 />
+</CodeGroup>
+
+import SharedPartitionByAPIOSS from '/snippets/ingest-configuration-shared/partition-by-api-oss.mdx';
+
+<SharedPartitionByAPIOSS/>

snippets/general-shared-text/discord-cli-api.mdx

@@ -0,0 +1,13 @@
+The Discord connector dependencies:
+
+```bash CLI, Python
+pip install "unstructured-ingest[discord]"
+```
+
+import AdditionalIngestDependencies from '/snippets/general-shared-text/ingest-dependencies.mdx';
+
+<AdditionalIngestDependencies />
+
+The following environment variables:
+
+- `DISCORD_ACCESS_TOKEN` - The Discord access token, represented by `--token` (CLI) or `token` (Python).

snippets/general-shared-text/discord.mdx

@@ -0,0 +1,2 @@
+- A Discord [access token](https://discord.com/developers/docs/topics/oauth2).
+- The target Discord [channel IDs](https://support.discord.com/hc/articles/206346498-Where-can-I-find-my-User-Server-Message-ID#h_01HRSTXPS5FMK2A5SMVSX4JW4E).

snippets/sc-shared-text/discord-cli-api.mdx

@@ -0,0 +1,9 @@
+Connect Discord to your preprocessing pipeline, and use the Unstructured CLI or Python to batch process all your documents and store structured outputs locally on your filesystem.
+
+The requirements are as follows. 
+
+import SharedDiscord from '/snippets/general-shared-text/discord.mdx';
+import SharedDiscordCLIAPI from '/snippets/general-shared-text/discord-cli-api.mdx';
+
+<SharedDiscord />
+<SharedDiscordCLIAPI />

snippets/source_connectors/discord.sh.mdx

@@ -0,0 +1,12 @@
+```bash CLI
+#!/usr/bin/env bash
+
+unstructured-ingest \
+  discord \
+    --token $DISCORD_ACCESS_TOKEN \
+    --channels 475182341782896651,847950191108554762 \
+    --output-dir $LOCAL_FILE_OUTPUT_DIR \
+    --partition-by-api \
+    --api-key $UNSTRUCTURED_API_KEY \
+    --partition-endpoint $UNSTRUCTURED_API_URL
+```

snippets/source_connectors/discord.v2.py.mdx

@@ -0,0 +1,49 @@
+```python Python Ingest
+import os
+
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.interfaces import ProcessorConfig
+
+from unstructured_ingest.processes.connectors.discord import (
+    DiscordIndexerConfig,
+    DiscordDownloaderConfig,
+    DiscordConnectionConfig,
+    DiscordAccessConfig
+)
+
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.embedder import EmbedderConfig
+
+from unstructured_ingest.processes.connectors.local import LocalUploaderConfig
+
+# Chunking and embedding are optional.
+
+if __name__ == "__main__":
+    Pipeline.from_configs(
+        context=ProcessorConfig(),
+        indexer_config=DiscordIndexerConfig(
+            channels=[
+                "475182341782896651",
+                "847950191108554762"
+            ]
+        ),
+        downloader_config=DiscordDownloaderConfig(download_dir=os.getenv("LOCAL_FILE_DOWNLOAD_DIR")),
+        source_connection_config=DiscordConnectionConfig(
+            access_config=DiscordAccessConfig(token=os.getenv("DISCORD_ACCESS_TOKEN"))
+        ),
+        partitioner_config=PartitionerConfig(
+            partition_by_api=True,
+            api_key=os.getenv("UNSTRUCTURED_API_KEY"),
+            partition_endpoint=os.getenv("UNSTRUCTURED_API_URL"),
+            additional_partition_args={
+                "split_pdf_page": True,
+                "split_pdf_allow_failed": True,
+                "split_pdf_concurrency_level": 15
+            }
+        ),
+        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        uploader_config=LocalUploaderConfig(output_dir=os.getenv("LOCAL_FILE_OUTPUT_DIR"))
+    ).run()
+```